From 667d8b8dd042ed4a8a510a83aa491b0f046bc599 Mon Sep 17 00:00:00 2001 From: Peter Bourgon Date: Tue, 9 Jun 2015 17:25:00 +0200 Subject: [PATCH] OriginHosts and OriginNodes become Origins Another baby step towards #123, this change follows from #192 and merges the two concepts of Origin in a renderable node. We also cut out a layer of abstraction, and add an OriginTable method to Report, which directly generates a table of info for the detail pane given any origin node ID. Other changes from feedback: - Assume origin IDs are unique and don't reflect.Dedupe - Improve origin ID lookup - Move OriginTable to detailed_node.go, as a free function - rm app/detail_pane.go (empty) --- app/api_topology.go | 4 +- app/api_topology_test.go | 12 +++- app/detail_pane.go | 94 ----------------------------- report/detailed_node.go | 107 +++++++++++++++++++++++++++++++++ report/detailed_node_test.go | 7 +++ report/report.go | 17 +++--- report/topology.go | 21 ++++--- report/topology_test.go | 113 ++++++++++++++++------------------- 8 files changed, 195 insertions(+), 180 deletions(-) delete mode 100644 app/detail_pane.go create mode 100644 report/detailed_node.go create mode 100644 report/detailed_node_test.go diff --git a/app/api_topology.go b/app/api_topology.go index e44b3e3299..282d4484b6 100644 --- a/app/api_topology.go +++ b/app/api_topology.go @@ -66,9 +66,7 @@ func handleNode(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Req http.NotFound(w, r) return } - originHostFunc := func(id string) (OriginHost, bool) { return getOriginHost(rpt.Host, id) } - originNodeFunc := func(id string) (OriginNode, bool) { return getOriginNode(t.selector(rpt), id) } - respondWith(w, http.StatusOK, APINode{Node: makeDetailed(node, originHostFunc, originNodeFunc)}) + respondWith(w, http.StatusOK, APINode{Node: report.MakeDetailedNode(rpt, node)}) } // Individual edges. diff --git a/app/api_topology_test.go b/app/api_topology_test.go index c540458c4a..f8f0729d14 100644 --- a/app/api_topology_test.go +++ b/app/api_topology_test.go @@ -28,7 +28,12 @@ func TestAPITopologyApplications(t *testing.T) { } equals(t, 1, len(node.Adjacency)) equals(t, report.MakeIDList("pid:node-b.local:215"), node.Adjacency) - equals(t, report.MakeIDList("hostA"), node.OriginHosts) + equals(t, report.MakeIDList( + report.MakeEndpointNodeID("hostA", "192.168.1.1", "12345"), + report.MakeEndpointNodeID("hostA", "192.168.1.1", "12346"), + report.MakeHostNodeID("hostA"), + ), node.Origins, + ) equals(t, "curl", node.LabelMajor) equals(t, "node-a.local (23128)", node.LabelMinor) equals(t, "23128", node.Rank) @@ -79,7 +84,10 @@ func TestAPITopologyHosts(t *testing.T) { t.Errorf("missing host:host-b node") } equals(t, report.MakeIDList("host:host-a"), node.Adjacency) - equals(t, report.MakeIDList("hostB"), node.OriginHosts) + equals(t, report.MakeIDList( + report.MakeAddressNodeID("hostB", "192.168.1.2"), + report.MakeHostNodeID("hostB"), + ), node.Origins) equals(t, "host-b", node.LabelMajor) equals(t, "", node.LabelMinor) equals(t, "host-b", node.Rank) diff --git a/app/detail_pane.go b/app/detail_pane.go deleted file mode 100644 index aaa1f84359..0000000000 --- a/app/detail_pane.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -import ( - "fmt" - "reflect" - "strconv" - - "github.com/weaveworks/scope/report" -) - -func makeDetailed( - n report.RenderableNode, - originHostLookup func(string) (OriginHost, bool), - originNodeLookup func(string) (OriginNode, bool), -) report.DetailedNode { - tables := []report.Table{{ - Title: "Connections", - Numeric: true, - Rows: []report.Row{ - // TODO omit these rows if there's no data? - {"TCP connections", strconv.FormatInt(int64(n.Metadata[report.KeyMaxConnCountTCP]), 10), ""}, - {"Bytes ingress", strconv.FormatInt(int64(n.Metadata[report.KeyBytesIngress]), 10), ""}, - {"Bytes egress", strconv.FormatInt(int64(n.Metadata[report.KeyBytesEgress]), 10), ""}, - }, - }} - - // Note that a RenderableNode may be the result of merge operation(s), and - // so may have multiple origin hosts and nodes. - -outer: - for _, id := range n.OriginNodes { - // Origin node IDs in e.g. the process topology are actually network - // n-tuples. (The process topology is actually more like a network - // n-tuple topology.) So we can have multiple IDs mapping to the same - // process. There are several ways to dedupe that, but here we take - // the lazy way and do simple equivalence of the resulting table. - node, ok := originNodeLookup(id) - if !ok { - node = unknownOriginNode(id) - } - for _, table := range tables { - if reflect.DeepEqual(table, node.Table) { - continue outer - } - } - tables = append(tables, node.Table) - } - - for _, id := range n.OriginHosts { - host, ok := originHostLookup(id) - if !ok { - host = unknownOriginHost(id) - } - tables = append(tables, report.Table{ - Title: "Origin Host", - Numeric: false, - Rows: []report.Row{ - {"Hostname", host.Hostname, ""}, - {"Load", host.Load, ""}, - {"OS", host.OS, ""}, - {"ID", id, ""}, - }, - }) - } - - return report.DetailedNode{ - ID: n.ID, - LabelMajor: n.LabelMajor, - LabelMinor: n.LabelMinor, - Pseudo: n.Pseudo, - Tables: tables, - } -} - -func unknownOriginHost(id string) OriginHost { - return OriginHost{ - Hostname: fmt.Sprintf("[%s]", id), - OS: "unknown", - Networks: []string{}, - Load: "", - } -} - -func unknownOriginNode(id string) OriginNode { - return OriginNode{ - Table: report.Table{ - Title: "Origin Node", - Numeric: false, - Rows: []report.Row{ - {"ID", id, ""}, - }, - }, - } -} diff --git a/report/detailed_node.go b/report/detailed_node.go new file mode 100644 index 0000000000..a841e7a98b --- /dev/null +++ b/report/detailed_node.go @@ -0,0 +1,107 @@ +package report + +import "strconv" + +// MakeDetailedNode transforms a renderable node to a detailed node. It uses +// aggregate metadata, plus the set of origin node IDs, to produce tables. +func MakeDetailedNode(r Report, n RenderableNode) DetailedNode { + tables := []Table{} + { + rows := []Row{} + if val, ok := n.Metadata[KeyMaxConnCountTCP]; ok { + rows = append(rows, Row{"TCP connections", strconv.FormatInt(int64(val), 10), ""}) + } + if val, ok := n.Metadata[KeyBytesIngress]; ok { + rows = append(rows, Row{"Bytes ingress", strconv.FormatInt(int64(val), 10), ""}) + } + if val, ok := n.Metadata[KeyBytesEgress]; ok { + rows = append(rows, Row{"Bytes egress", strconv.FormatInt(int64(val), 10), ""}) + } + if len(rows) > 0 { + tables = append(tables, Table{"Connections", true, rows}) + } + } + + // RenderableNode may be the result of merge operation(s), and so may have + // multiple origins. The ultimate goal here is to generate tables to view + // in the UI, so we skip the intermediate representations, but we could + // add them later. + for _, id := range n.Origins { + if table, ok := OriginTable(r, id); ok { + // Origin node IDs are unique, so we'll be optimistic, here, and + // assume they'll also produce unique tables. + tables = append(tables, table) + } + } + + return DetailedNode{ + ID: n.ID, + LabelMajor: n.LabelMajor, + LabelMinor: n.LabelMinor, + Pseudo: n.Pseudo, + Tables: tables, + } +} + +// OriginTable produces a table (to be consumed directly by the UI) based on +// an origin ID, which is (optimistically) a node ID in one of our topologies. +func OriginTable(r Report, originID string) (Table, bool) { + if nmd, ok := r.Endpoint.NodeMetadatas[originID]; ok { + return endpointOriginTable(nmd) + } + if nmd, ok := r.Address.NodeMetadatas[originID]; ok { + return addressOriginTable(nmd) + } + if nmd, ok := r.Host.NodeMetadatas[originID]; ok { + return hostOriginTable(nmd) + } + return Table{}, false +} + +func endpointOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["endpoint"]; ok { + rows = append(rows, Row{"Endpoint", val, ""}) + } + if val, ok := nmd["host_name"]; ok { + rows = append(rows, Row{"Host name", val, ""}) + } + return Table{ + Title: "Origin Endpoint", + Numeric: false, + Rows: rows, + }, len(rows) > 0 +} + +func addressOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["address"]; ok { + rows = append(rows, Row{"Address", val, ""}) + } + if val, ok := nmd["host_name"]; ok { + rows = append(rows, Row{"Host name", val, ""}) + } + return Table{ + Title: "Origin Address", + Numeric: false, + Rows: rows, + }, len(rows) > 0 +} + +func hostOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["host_name"]; ok { + rows = append(rows, Row{"Host name", val, ""}) + } + if val, ok := nmd["load"]; ok { + rows = append(rows, Row{"Load", val, ""}) + } + if val, ok := nmd["os"]; ok { + rows = append(rows, Row{"Operating system", val, ""}) + } + return Table{ + Title: "Origin Host", + Numeric: false, + Rows: rows, + }, len(rows) > 0 +} diff --git a/report/detailed_node_test.go b/report/detailed_node_test.go new file mode 100644 index 0000000000..b09304016c --- /dev/null +++ b/report/detailed_node_test.go @@ -0,0 +1,7 @@ +package report_test + +import "testing" + +func TestMakeDetailedNode(t *testing.T) { + t.Skip("TODO") +} diff --git a/report/report.go b/report/report.go index be63a555a6..c3b0827ebb 100644 --- a/report/report.go +++ b/report/report.go @@ -29,15 +29,14 @@ type Report struct { // an element of a topology. It should contain information that's relevant // to rendering a node when there are many nodes visible at once. type RenderableNode struct { - ID string `json:"id"` // - LabelMajor string `json:"label_major"` // e.g. "process", human-readable - LabelMinor string `json:"label_minor,omitempty"` // e.g. "hostname", human-readable, optional - Rank string `json:"rank"` // to help the layout engine - Pseudo bool `json:"pseudo,omitempty"` // sort-of a placeholder node, for rendering purposes - Adjacency IDList `json:"adjacency,omitempty"` // Node IDs (in the same topology domain) - OriginHosts IDList `json:"origin_hosts,omitempty"` // Which hosts contributed information to this node - OriginNodes IDList `json:"origin_nodes,omitempty"` // Which origin nodes (depends on topology) contributed - Metadata AggregateMetadata `json:"metadata"` // Numeric sums + ID string `json:"id"` // + LabelMajor string `json:"label_major"` // e.g. "process", human-readable + LabelMinor string `json:"label_minor,omitempty"` // e.g. "hostname", human-readable, optional + Rank string `json:"rank"` // to help the layout engine + Pseudo bool `json:"pseudo,omitempty"` // sort-of a placeholder node, for rendering purposes + Adjacency IDList `json:"adjacency,omitempty"` // Node IDs (in the same topology domain) + Origins IDList `json:"origins,omitempty"` // Core node IDs that contributed information + Metadata AggregateMetadata `json:"metadata"` // Numeric sums } // DetailedNode is the data type that's yielded to the JavaScript layer when diff --git a/report/topology.go b/report/topology.go index 1ad2c7393d..59527379f0 100644 --- a/report/topology.go +++ b/report/topology.go @@ -98,15 +98,14 @@ func (t Topology) RenderBy(mapFunc MapFunc, pseudoFunc PseudoFunc) map[string]Re // the existing data, on the assumption that the MapFunc returns the same // data. nodes[mapped.ID] = RenderableNode{ - ID: mapped.ID, - LabelMajor: mapped.Major, - LabelMinor: mapped.Minor, - Rank: mapped.Rank, - Pseudo: false, - Adjacency: IDList{}, // later - OriginHosts: IDList{}, // later - OriginNodes: IDList{}, // later - Metadata: AggregateMetadata{}, // later + ID: mapped.ID, + LabelMajor: mapped.Major, + LabelMinor: mapped.Minor, + Rank: mapped.Rank, + Pseudo: false, + Adjacency: IDList{}, // later + Origins: IDList{}, // later + Metadata: AggregateMetadata{}, // later } address2mapped[addressID] = mapped.ID } @@ -142,8 +141,8 @@ func (t Topology) RenderBy(mapFunc MapFunc, pseudoFunc PseudoFunc) map[string]Re } srcRenderableNode.Adjacency = srcRenderableNode.Adjacency.Add(dstRenderableID) - srcRenderableNode.OriginHosts = srcRenderableNode.OriginHosts.Add(srcOriginHostID) - srcRenderableNode.OriginNodes = srcRenderableNode.OriginNodes.Add(srcNodeAddress) + srcRenderableNode.Origins = srcRenderableNode.Origins.Add(MakeHostNodeID(srcOriginHostID)) + srcRenderableNode.Origins = srcRenderableNode.Origins.Add(srcNodeAddress) edgeID := MakeEdgeID(srcNodeAddress, dstNodeAddress) if md, ok := t.EdgeMetadatas[edgeID]; ok { srcRenderableNode.Metadata.Merge(md.Transform()) diff --git a/report/topology_test.go b/report/topology_test.go index 68c4180668..658cbf2890 100644 --- a/report/topology_test.go +++ b/report/topology_test.go @@ -141,28 +141,26 @@ var ( func TestRenderByEndpointPID(t *testing.T) { want := map[string]RenderableNode{ "pid:client-54001-domain:10001": { - ID: "pid:client-54001-domain:10001", - LabelMajor: "curl", - LabelMinor: "client-54001-domain (10001)", - Rank: "10001", - Pseudo: false, - Adjacency: MakeIDList("pid:server-80-domain:215"), - OriginHosts: MakeIDList("client.hostname.com"), - OriginNodes: MakeIDList("client.hostname.com;10.10.10.20;54001"), + ID: "pid:client-54001-domain:10001", + LabelMajor: "curl", + LabelMinor: "client-54001-domain (10001)", + Rank: "10001", + Pseudo: false, + Adjacency: MakeIDList("pid:server-80-domain:215"), + Origins: MakeIDList(MakeHostNodeID("client.hostname.com"), MakeEndpointNodeID("client.hostname.com", "10.10.10.20", "54001")), Metadata: AggregateMetadata{ KeyBytesIngress: 100, KeyBytesEgress: 10, }, }, "pid:client-54002-domain:10001": { - ID: "pid:client-54002-domain:10001", - LabelMajor: "curl", - LabelMinor: "client-54002-domain (10001)", - Rank: "10001", // same process - Pseudo: false, - Adjacency: MakeIDList("pid:server-80-domain:215"), - OriginHosts: MakeIDList("client.hostname.com"), - OriginNodes: MakeIDList("client.hostname.com;10.10.10.20;54002"), + ID: "pid:client-54002-domain:10001", + LabelMajor: "curl", + LabelMinor: "client-54002-domain (10001)", + Rank: "10001", // same process + Pseudo: false, + Adjacency: MakeIDList("pid:server-80-domain:215"), + Origins: MakeIDList(MakeHostNodeID("client.hostname.com"), MakeEndpointNodeID("client.hostname.com", "10.10.10.20", "54002")), Metadata: AggregateMetadata{ KeyBytesIngress: 200, KeyBytesEgress: 20, @@ -180,8 +178,7 @@ func TestRenderByEndpointPID(t *testing.T) { "pseudo;10.10.10.10;192.168.1.1;80", "pseudo;10.10.10.11;192.168.1.1;80", ), - OriginHosts: MakeIDList("server.hostname.com"), - OriginNodes: MakeIDList("server.hostname.com;192.168.1.1;80"), + Origins: MakeIDList(MakeHostNodeID("server.hostname.com"), MakeEndpointNodeID("server.hostname.com", "192.168.1.1", "80")), Metadata: AggregateMetadata{ KeyBytesIngress: 150, KeyBytesEgress: 1500, @@ -212,14 +209,13 @@ func TestRenderByEndpointPIDGrouped(t *testing.T) { // dimensions from the ID. That could be changed. want := map[string]RenderableNode{ "curl": { - ID: "curl", - LabelMajor: "curl", - LabelMinor: "", - Rank: "curl", - Pseudo: false, - Adjacency: MakeIDList("apache"), - OriginHosts: MakeIDList("client.hostname.com"), - OriginNodes: MakeIDList("client.hostname.com;10.10.10.20;54001", "client.hostname.com;10.10.10.20;54002"), + ID: "curl", + LabelMajor: "curl", + LabelMinor: "", + Rank: "curl", + Pseudo: false, + Adjacency: MakeIDList("apache"), + Origins: MakeIDList(MakeHostNodeID("client.hostname.com"), MakeEndpointNodeID("client.hostname.com", "10.10.10.20", "54001"), MakeEndpointNodeID("client.hostname.com", "10.10.10.20", "54002")), Metadata: AggregateMetadata{ KeyBytesIngress: 300, KeyBytesEgress: 30, @@ -236,8 +232,7 @@ func TestRenderByEndpointPIDGrouped(t *testing.T) { "pseudo;10.10.10.10;apache", "pseudo;10.10.10.11;apache", ), - OriginHosts: MakeIDList("server.hostname.com"), - OriginNodes: MakeIDList("server.hostname.com;192.168.1.1;80"), + Origins: MakeIDList(MakeHostNodeID("server.hostname.com"), MakeEndpointNodeID("server.hostname.com", "192.168.1.1", "80")), Metadata: AggregateMetadata{ KeyBytesIngress: 150, KeyBytesEgress: 1500, @@ -265,54 +260,50 @@ func TestRenderByEndpointPIDGrouped(t *testing.T) { func TestRenderByNetworkHostname(t *testing.T) { want := map[string]RenderableNode{ "host:client.hostname.com": { - ID: "host:client.hostname.com", - LabelMajor: "client", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "client", - Pseudo: false, - Adjacency: MakeIDList("host:server.hostname.com"), - OriginHosts: MakeIDList("client.hostname.com"), - OriginNodes: MakeIDList("client.hostname.com;10.10.10.20"), + ID: "host:client.hostname.com", + LabelMajor: "client", // before first . + LabelMinor: "hostname.com", // after first . + Rank: "client", + Pseudo: false, + Adjacency: MakeIDList("host:server.hostname.com"), + Origins: MakeIDList(MakeHostNodeID("client.hostname.com"), MakeAddressNodeID("client.hostname.com", "10.10.10.20")), Metadata: AggregateMetadata{ KeyMaxConnCountTCP: 3, }, }, "host:random.hostname.com": { - ID: "host:random.hostname.com", - LabelMajor: "random", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "random", - Pseudo: false, - Adjacency: MakeIDList("host:server.hostname.com"), - OriginHosts: MakeIDList("random.hostname.com"), - OriginNodes: MakeIDList("random.hostname.com;172.16.11.9"), + ID: "host:random.hostname.com", + LabelMajor: "random", // before first . + LabelMinor: "hostname.com", // after first . + Rank: "random", + Pseudo: false, + Adjacency: MakeIDList("host:server.hostname.com"), + Origins: MakeIDList(MakeHostNodeID("random.hostname.com"), MakeAddressNodeID("random.hostname.com", "172.16.11.9")), Metadata: AggregateMetadata{ KeyMaxConnCountTCP: 20, }, }, "host:server.hostname.com": { - ID: "host:server.hostname.com", - LabelMajor: "server", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "server", - Pseudo: false, - Adjacency: MakeIDList("host:client.hostname.com", "pseudo;10.10.10.10;192.168.1.1;"), - OriginHosts: MakeIDList("server.hostname.com"), - OriginNodes: MakeIDList("server.hostname.com;192.168.1.1"), + ID: "host:server.hostname.com", + LabelMajor: "server", // before first . + LabelMinor: "hostname.com", // after first . + Rank: "server", + Pseudo: false, + Adjacency: MakeIDList("host:client.hostname.com", "pseudo;10.10.10.10;192.168.1.1;"), + Origins: MakeIDList(MakeHostNodeID("server.hostname.com"), MakeAddressNodeID("server.hostname.com", "192.168.1.1")), Metadata: AggregateMetadata{ KeyMaxConnCountTCP: 10, }, }, "pseudo;10.10.10.10;192.168.1.1;": { - ID: "pseudo;10.10.10.10;192.168.1.1;", - LabelMajor: "10.10.10.10", - LabelMinor: "", // after first . - Rank: "", - Pseudo: true, - Adjacency: nil, - OriginHosts: nil, - OriginNodes: nil, - Metadata: AggregateMetadata{}, + ID: "pseudo;10.10.10.10;192.168.1.1;", + LabelMajor: "10.10.10.10", + LabelMinor: "", // after first . + Rank: "", + Pseudo: true, + Adjacency: nil, + Origins: nil, + Metadata: AggregateMetadata{}, }, } have := report.Address.RenderBy(NetworkHostname, GenericPseudoNode)