diff --git a/Makefile b/Makefile index de955c4878..32e6016a9f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SCOPE_UI_BUILD_IMAGE=$(DOCKERHUB_USER)/scope-ui-build all: $(SCOPE_EXPORT) -$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) docker/* +$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) docker/* cp $(APP_EXE) $(PROBE_EXE) docker/ $(SUDO) docker build -t $(SCOPE_IMAGE) docker/ $(SUDO) docker save $(SCOPE_IMAGE):latest | $(SUDO) $(DOCKER_SQUASH) -t $(SCOPE_IMAGE) | tee $@ | $(SUDO) docker load @@ -45,8 +45,11 @@ clean: rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_EXPORT) client/dist deps: - go get github.com/jwilder/docker-squash \ + go get \ + github.com/jwilder/docker-squash \ github.com/golang/lint/golint \ github.com/fzipp/gocyclo \ github.com/mattn/goveralls \ - github.com/mjibson/esc + github.com/mjibson/esc \ + github.com/davecgh/go-spew/spew \ + github.com/pmezard/go-difflib/difflib diff --git a/app/api_topology.go b/app/api_topology.go index 6098932572..7431ff3113 100644 --- a/app/api_topology.go +++ b/app/api_topology.go @@ -1,9 +1,7 @@ package main import ( - "fmt" "net/http" - "strconv" "time" "github.com/gorilla/mux" @@ -54,12 +52,9 @@ func makeTopologyHandlers( ) { // Full topology. get.HandleFunc(base, func(w http.ResponseWriter, r *http.Request) { - rpt := rep.Report() - rendered := topo(rpt).RenderBy(mapping, grouped) - t := APITopology{ - Nodes: rendered, - } - respondWith(w, http.StatusOK, t) + respondWith(w, http.StatusOK, APITopology{ + Nodes: topo(rep.Report()).RenderBy(mapping, grouped), + }) }) // Websocket for the full topology. This route overlaps with the next. @@ -88,8 +83,9 @@ func makeTopologyHandlers( http.NotFound(w, r) return } - of := func(nodeID string) (Origin, bool) { return getOrigin(rep, nodeID) } - respondWith(w, http.StatusOK, APINode{Node: makeDetailed(node, of)}) + originHostFunc := func(id string) (OriginHost, bool) { return getOriginHost(rpt.HostMetadatas, id) } + originNodeFunc := func(id string) (OriginNode, bool) { return getOriginNode(topo(rpt), id) } + respondWith(w, http.StatusOK, APINode{Node: makeDetailed(node, originHostFunc, originNodeFunc)}) }) // Individual edges. @@ -105,48 +101,6 @@ func makeTopologyHandlers( }) } -// TODO(pb): temporary hack -func makeDetailed(n report.RenderableNode, of func(string) (Origin, bool)) report.DetailedNode { - // A RenderableNode may be the result of merge operation(s), and so may - // have multiple origins. - origins := []report.Table{} - for _, originID := range n.Origin { - origin, ok := of(originID) - if !ok { - origin = unknownOrigin - } - origins = append(origins, report.Table{ - Title: "Origin", - Numeric: false, - Rows: []report.Row{ - {"Hostname", origin.Hostname, ""}, - {"Load", fmt.Sprintf("%.2f %.2f %.2f", origin.LoadOne, origin.LoadFive, origin.LoadFifteen), ""}, - {"OS", origin.OS, ""}, - //{"Addresses", strings.Join(origin.Addresses, ", "), ""}, - {"ID", originID, ""}, - }, - }) - } - - tables := []report.Table{} - tables = append(tables, report.Table{ - Title: "Connections", - Numeric: true, - Rows: []report.Row{ - {"TCP (max)", strconv.FormatInt(int64(n.Metadata[report.KeyMaxConnCountTCP]), 10), ""}, - }, - }) - tables = append(tables, origins...) - - return report.DetailedNode{ - ID: n.ID, - LabelMajor: n.LabelMajor, - LabelMinor: n.LabelMinor, - Pseudo: n.Pseudo, - Tables: tables, - } -} - var upgrader = websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { return true }, } @@ -169,8 +123,7 @@ func handleWebsocket( quit := make(chan struct{}) go func(c *websocket.Conn) { - // Discard all the browser sends us. - for { + for { // just discard everything the browser sends if _, _, err := c.NextReader(); err != nil { close(quit) break @@ -178,7 +131,10 @@ func handleWebsocket( } }(conn) - var previousTopo map[string]report.RenderableNode + var ( + previousTopo map[string]report.RenderableNode + tick = time.Tick(loop) + ) for { newTopo := topo(rep.Report()).RenderBy(mapping, grouped) diff := report.TopoDiff(previousTopo, newTopo) @@ -192,16 +148,7 @@ func handleWebsocket( select { case <-quit: return - case <-time.After(loop): + case <-tick: } } } - -var unknownOrigin = Origin{ - Hostname: "unknown", - OS: "unknown", - Addresses: []string{}, - LoadOne: 0.0, - LoadFive: 0.0, - LoadFifteen: 0.0, -} diff --git a/app/api_topology_test.go b/app/api_topology_test.go index 203ddbd588..152d722bfd 100644 --- a/app/api_topology_test.go +++ b/app/api_topology_test.go @@ -14,14 +14,12 @@ import ( func TestAPITopologyApplications(t *testing.T) { ts := httptest.NewServer(Router(StaticReport{})) defer ts.Close() - is404(t, ts, "/api/topology/applications/foobar") - { body := getRawJSON(t, ts, "/api/topology/applications") var topo APITopology if err := json.Unmarshal(body, &topo); err != nil { - t.Fatalf("JSON parse error: %s", err) + t.Fatal(err) } equals(t, 4, len(topo.Nodes)) node, ok := topo.Nodes["pid:node-a.local:23128"] @@ -30,25 +28,25 @@ func TestAPITopologyApplications(t *testing.T) { } equals(t, 1, len(node.Adjacency)) equals(t, report.NewIDList("pid:node-b.local:215"), node.Adjacency) - equals(t, report.NewIDList("hostA"), node.Origin) + equals(t, report.NewIDList("hostA"), node.OriginHosts) equals(t, "curl", node.LabelMajor) equals(t, "node-a.local (23128)", node.LabelMinor) equals(t, "23128", node.Rank) equals(t, false, node.Pseudo) } - { - // Node detail body := getRawJSON(t, ts, "/api/topology/applications/pid:node-a.local:23128") var node APINode if err := json.Unmarshal(body, &node); err != nil { - t.Fatalf("JSON parse error: %s", err) + t.Fatal(err) } - // TODO(pb): replace + equals(t, "pid:node-a.local:23128", node.Node.ID) + equals(t, "curl", node.Node.LabelMajor) + equals(t, "node-a.local (23128)", node.Node.LabelMinor) + equals(t, false, node.Node.Pseudo) + // Let's not unit-test the specific content of the detail tables } - { - // Edge detail body := getRawJSON(t, ts, "/api/topology/applications/pid:node-a.local:23128/pid:node-b.local:215") var edge APIEdge if err := json.Unmarshal(body, &edge); err != nil { @@ -68,41 +66,38 @@ func TestAPITopologyApplications(t *testing.T) { func TestAPITopologyHosts(t *testing.T) { ts := httptest.NewServer(Router(StaticReport{})) defer ts.Close() - is404(t, ts, "/api/topology/hosts/foobar") - { body := getRawJSON(t, ts, "/api/topology/hosts") var topo APITopology if err := json.Unmarshal(body, &topo); err != nil { - t.Fatalf("JSON parse error: %s", err) + t.Fatal(err) } - equals(t, 3, len(topo.Nodes)) node, ok := topo.Nodes["host:host-b"] if !ok { t.Errorf("missing host:host-b node") } equals(t, report.NewIDList("host:host-a"), node.Adjacency) - equals(t, report.NewIDList("hostB"), node.Origin) + equals(t, report.NewIDList("hostB"), node.OriginHosts) equals(t, "host-b", node.LabelMajor) equals(t, "", node.LabelMinor) equals(t, "host-b", node.Rank) equals(t, false, node.Pseudo) } - { - // Node detail body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b") var node APINode if err := json.Unmarshal(body, &node); err != nil { - t.Fatalf("JSON parse error: %s", err) + t.Fatal(err) } - // TODO(pb): replace + equals(t, "host:host-b", node.Node.ID) + equals(t, "host-b", node.Node.LabelMajor) + equals(t, "", node.Node.LabelMinor) + equals(t, false, node.Node.Pseudo) + // Let's not unit-test the specific content of the detail tables } - { - // Edge detail body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b/host:host-a") var edge APIEdge if err := json.Unmarshal(body, &edge); err != nil { @@ -123,24 +118,23 @@ func TestAPITopologyHosts(t *testing.T) { func TestAPITopologyWebsocket(t *testing.T) { ts := httptest.NewServer(Router(StaticReport{})) defer ts.Close() - url := "/api/topology/applications/ws" - // Not a websocket request: + // Not a websocket request res, _ := checkGet(t, ts, url) if have := res.StatusCode; have != 400 { t.Fatalf("Expected status %d, got %d.", 400, have) } - // Proper websocket request: + // Proper websocket request ts.URL = "ws" + ts.URL[len("http"):] dialer := &websocket.Dialer{} ws, res, err := dialer.Dial(ts.URL+url, nil) ok(t, err) defer ws.Close() - if have := res.StatusCode; have != 101 { - t.Fatalf("Expected status %d, got %d.", 101, have) + if want, have := 101, res.StatusCode; want != have { + t.Fatalf("want %d, have %d", want, have) } _, p, err := ws.ReadMessage() diff --git a/app/detail_pane.go b/app/detail_pane.go new file mode 100644 index 0000000000..2f607f2f27 --- /dev/null +++ b/app/detail_pane.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "reflect" + "strconv" + + "github.com/weaveworks/scope/report" +) + +func makeDetailed( + n report.RenderableNode, + originHostLookup func(string) (OriginHost, bool), + originNodeLookup func(string) (OriginNode, bool), +) report.DetailedNode { + tables := []report.Table{{ + Title: "Connections", + Numeric: true, + Rows: []report.Row{ + // TODO omit these rows if there's no data? + {"TCP connections", strconv.FormatInt(int64(n.Metadata[report.KeyMaxConnCountTCP]), 10), ""}, + {"Bytes ingress", strconv.FormatInt(int64(n.Metadata[report.KeyBytesIngress]), 10), ""}, + {"Bytes egress", strconv.FormatInt(int64(n.Metadata[report.KeyBytesEgress]), 10), ""}, + }, + }} + + // Note that a RenderableNode may be the result of merge operation(s), and + // so may have multiple origin hosts and nodes. + +outer: + for _, id := range n.OriginNodes { + // Origin node IDs in e.g. the process topology are actually network + // n-tuples. (The process topology is actually more like a network + // n-tuple topology.) So we can have multiple IDs mapping to the same + // process. There are several ways to dedupe that, but here we take + // the lazy way and do simple equivalence of the resulting table. + node, ok := originNodeLookup(id) + if !ok { + node = unknownOriginNode(id) + } + for _, table := range tables { + if reflect.DeepEqual(table, node.Table) { + continue outer + } + } + tables = append(tables, node.Table) + } + + for _, id := range n.OriginHosts { + host, ok := originHostLookup(id) + if !ok { + host = unknownOriginHost(id) + } + tables = append(tables, report.Table{ + Title: "Origin Host", + Numeric: false, + Rows: []report.Row{ + {"Hostname", host.Hostname, ""}, + {"Load", fmt.Sprintf("%.2f %.2f %.2f", host.LoadOne, host.LoadFive, host.LoadFifteen), ""}, + {"OS", host.OS, ""}, + //{"Addresses", strings.Join(host.Addresses, ", "), ""}, + {"ID", id, ""}, + }, + }) + } + + return report.DetailedNode{ + ID: n.ID, + LabelMajor: n.LabelMajor, + LabelMinor: n.LabelMinor, + Pseudo: n.Pseudo, + Tables: tables, + } +} + +func unknownOriginHost(id string) OriginHost { + return OriginHost{ + Hostname: fmt.Sprintf("[%s]", id), + OS: "unknown", + Addresses: []string{}, + LoadOne: 0.0, + LoadFive: 0.0, + LoadFifteen: 0.0, + } +} + +func unknownOriginNode(id string) OriginNode { + return OriginNode{ + Table: report.Table{ + Title: "Origin Node", + Numeric: false, + Rows: []report.Row{ + {"ID", id, ""}, + }, + }, + } +} diff --git a/app/api_origin.go b/app/origin_host.go similarity index 58% rename from app/api_origin.go rename to app/origin_host.go index e0aea23af5..a7b722ee60 100644 --- a/app/api_origin.go +++ b/app/origin_host.go @@ -4,11 +4,14 @@ import ( "net/http" "github.com/gorilla/mux" + + "github.com/weaveworks/scope/report" ) -// Origin is returned by the /api/origin/* handler. It represents a machine -// that runs a probe, i.e. the origin of some data in the system. -type Origin struct { +// OriginHost represents a host that runs a probe, i.e. the origin host of +// some data in the system. The struct is returned by the /api/origin/{id} +// handler. +type OriginHost struct { Hostname string `json:"hostname"` OS string `json:"os"` Addresses []string `json:"addresses"` @@ -17,26 +20,10 @@ type Origin struct { LoadFifteen float64 `json:"load_fifteen"` } -// makeOriginHandler makes the /api/origin/* handler. -func makeOriginHandler(rep Reporter) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - var ( - vars = mux.Vars(r) - nodeID = vars["id"] - ) - origin, ok := getOrigin(rep, nodeID) - if !ok { - http.NotFound(w, r) - return - } - respondWith(w, http.StatusOK, origin) - } -} - -func getOrigin(rep Reporter, nodeID string) (Origin, bool) { - host, ok := rep.Report().HostMetadatas[nodeID] +func getOriginHost(mds report.HostMetadatas, nodeID string) (OriginHost, bool) { + host, ok := mds[nodeID] if !ok { - return Origin{}, false + return OriginHost{}, false } var addrs []string @@ -44,7 +31,7 @@ func getOrigin(rep Reporter, nodeID string) (Origin, bool) { addrs = append(addrs, l.String()) } - return Origin{ + return OriginHost{ Hostname: host.Hostname, OS: host.OS, Addresses: addrs, @@ -53,3 +40,19 @@ func getOrigin(rep Reporter, nodeID string) (Origin, bool) { LoadFifteen: host.LoadFifteen, }, true } + +// makeOriginHostHandler makes the /api/origin/* handler. +func makeOriginHostHandler(rep Reporter) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + vars = mux.Vars(r) + nodeID = vars["id"] + ) + origin, ok := getOriginHost(rep.Report().HostMetadatas, nodeID) + if !ok { + http.NotFound(w, r) + return + } + respondWith(w, http.StatusOK, origin) + } +} diff --git a/app/api_origin_test.go b/app/origin_host_test.go similarity index 83% rename from app/api_origin_test.go rename to app/origin_host_test.go index 1f7bd30bcd..6ff9830413 100644 --- a/app/api_origin_test.go +++ b/app/origin_host_test.go @@ -6,16 +6,17 @@ import ( "testing" ) -func TestAPIOrigin(t *testing.T) { +func TestAPIOriginHost(t *testing.T) { ts := httptest.NewServer(Router(StaticReport{})) defer ts.Close() is404(t, ts, "/api/origin/foobar") + is404(t, ts, "/api/origin/host/foobar") { // Origin - body := getRawJSON(t, ts, "/api/origin/hostA") - var o Origin + body := getRawJSON(t, ts, "/api/origin/host/hostA") + var o OriginHost if err := json.Unmarshal(body, &o); err != nil { t.Fatalf("JSON parse error: %s", err) } diff --git a/app/origin_node.go b/app/origin_node.go new file mode 100644 index 0000000000..2cfa1b7700 --- /dev/null +++ b/app/origin_node.go @@ -0,0 +1,65 @@ +package main + +import "github.com/weaveworks/scope/report" + +// OriginNode is a node in the originating report topology. It's a process ID +// or network host. It's used by the /api/topology/{topology}/{nodeID} handler +// to generate detailed information. One node from a rendered topology may +// have multiple origin nodes. +type OriginNode struct { + Table report.Table +} + +func getOriginNode(t report.Topology, id string) (OriginNode, bool) { + node, ok := t.NodeMetadatas[id] + if !ok { + return OriginNode{}, false + } + + // The node represents different actual things depending on the topology. + // So we deduce what it is, based on the metadata. + if _, ok := node["pid"]; ok { + return originNodeForProcess(node), true + } + + // Assume network host. Could strengthen this guess by adding a + // special key in the probe spying procedure. + return originNodeForNetworkHost(node), true +} + +func originNodeForProcess(node report.NodeMetadata) OriginNode { + rows := []report.Row{ + {Key: "Host", ValueMajor: node["domain"], ValueMinor: ""}, + {Key: "PID", ValueMajor: node["pid"], ValueMinor: ""}, + {Key: "Process name", ValueMajor: node["name"], ValueMinor: ""}, + } + for key, human := range map[string]string{ + "docker_id": "Container ID", + "docker_name": "Container name", + "cgroup": "cgroup", + } { + if val, ok := node[key]; ok { + rows = append(rows, report.Row{Key: human, ValueMajor: val, ValueMinor: ""}) + } + } + return OriginNode{ + Table: report.Table{ + Title: "Origin Process", + Numeric: false, + Rows: rows, + }, + } +} + +func originNodeForNetworkHost(node report.NodeMetadata) OriginNode { + rows := []report.Row{ + {"Hostname", node["name"], ""}, + } + return OriginNode{ + Table: report.Table{ + Title: "Origin Host", + Numeric: false, + Rows: rows, + }, + } +} diff --git a/app/router.go b/app/router.go index d121a7872b..23772f9ceb 100644 --- a/app/router.go +++ b/app/router.go @@ -34,7 +34,7 @@ func Router(c Reporter) *mux.Router { ) } } - get.HandleFunc("/api/origin/{id}", makeOriginHandler(c)) + get.HandleFunc("/api/origin/host/{id}", makeOriginHostHandler(c)) get.HandleFunc("/api/report", makeRawReportHandler(c)) get.PathPrefix("/").Handler(http.FileServer(FS(false))) // everything else is static return router diff --git a/app/scope_test.go b/app/scope_test.go index ee1b3616f0..a9d9caa943 100644 --- a/app/scope_test.go +++ b/app/scope_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "fmt" "io" "io/ioutil" "net/http" @@ -17,26 +16,23 @@ import ( func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { if !condition { _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: "+msg+"\n", append([]interface{}{filepath.Base(file), line}, v...)...) - tb.FailNow() + tb.Fatalf("%s:%d: "+msg, append([]interface{}{filepath.Base(file), line}, v...)...) } } -// ok fails the test if an err is not nil. +// ok errors the test if an err is not nil. func ok(tb testing.TB, err error) { if err != nil { _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: unexpected error: %s\n", filepath.Base(file), line, err.Error()) - tb.FailNow() + tb.Errorf("%s:%d: unexpected error: %v", filepath.Base(file), line, err) } } -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}) { - if !reflect.DeepEqual(exp, act) { +// equals errors the test if want is not equal to have. +func equals(tb testing.TB, want, have interface{}) { + if !reflect.DeepEqual(want, have) { _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: expected: %#v got: %#v\n", filepath.Base(file), line, exp, act) - tb.FailNow() + tb.Errorf("%s:%d: want %#v, have %#v", filepath.Base(file), line, want, have) } } diff --git a/experimental/integration/cmd.go b/experimental/_integration/cmd.go similarity index 100% rename from experimental/integration/cmd.go rename to experimental/_integration/cmd.go diff --git a/experimental/integration/contexts.go b/experimental/_integration/contexts.go similarity index 100% rename from experimental/integration/contexts.go rename to experimental/_integration/contexts.go diff --git a/experimental/integration/doc.go b/experimental/_integration/doc.go similarity index 100% rename from experimental/integration/doc.go rename to experimental/_integration/doc.go diff --git a/experimental/integration/easy_test.go b/experimental/_integration/easy_test.go similarity index 100% rename from experimental/integration/easy_test.go rename to experimental/_integration/easy_test.go diff --git a/experimental/integration/no_test b/experimental/_integration/no_test similarity index 100% rename from experimental/integration/no_test rename to experimental/_integration/no_test diff --git a/experimental/integration/test_extra_report.json b/experimental/_integration/test_extra_report.json similarity index 100% rename from experimental/integration/test_extra_report.json rename to experimental/_integration/test_extra_report.json diff --git a/experimental/integration/test_single_report.json b/experimental/_integration/test_single_report.json similarity index 100% rename from experimental/integration/test_single_report.json rename to experimental/_integration/test_single_report.json diff --git a/experimental/graphviz/handle.go b/experimental/graphviz/handle.go index 39ae2ac049..c30cee2ea7 100644 --- a/experimental/graphviz/handle.go +++ b/experimental/graphviz/handle.go @@ -97,11 +97,11 @@ func engine(r *http.Request) string { } func mapFunc(r *http.Request) report.MapFunc { - f, ok := report.MapFuncRegistry[strings.ToLower(r.FormValue("map_func"))] - if !ok { - f = report.ProcessName + switch strings.ToLower(r.FormValue("map_func")) { + case "hosts", "networkhost", "networkhostname": + return report.NetworkHostname } - return f + return report.ProcessPID } func classView(r *http.Request) bool { diff --git a/probe/process_mapper_test.go b/probe/process_mapper_test.go index ba71752e1f..61f9694280 100644 --- a/probe/process_mapper_test.go +++ b/probe/process_mapper_test.go @@ -39,7 +39,7 @@ func setupTmpFS(t *testing.T, fs map[string]string) string { if err != nil { t.Fatal(err) } - t.Logf("using TempDir %s", tmp) + //t.Logf("using TempDir %s", tmp) for file, content := range fs { dir := path.Dir(file) diff --git a/probe/spy.go b/probe/spy.go index dfac75e746..422f205de4 100644 --- a/probe/spy.go +++ b/probe/spy.go @@ -16,7 +16,7 @@ import ( // of host and port. It optionally enriches that topology with process (PID) // information. func spy( - nodeID, nodeName string, + hostID, hostName string, includeProcesses bool, pms []processMapper, ) report.Report { @@ -33,7 +33,7 @@ func spy( } for conn := conns.Next(); conn != nil; conn = conns.Next() { - addConnection(&r, conn, nodeID, nodeName, pms) + addConnection(&r, conn, hostID, hostName, pms) } return r @@ -42,13 +42,13 @@ func spy( func addConnection( r *report.Report, c *procspy.Connection, - nodeID, nodeName string, + hostID, hostName string, pms []processMapper, ) { var ( - scopedLocal = scopedIP(nodeID, c.LocalAddress) - scopedRemote = scopedIP(nodeID, c.RemoteAddress) - key = nodeID + report.IDDelim + scopedLocal + scopedLocal = scopedIP(hostID, c.LocalAddress) + scopedRemote = scopedIP(hostID, c.RemoteAddress) + key = hostID + report.IDDelim + scopedLocal edgeKey = scopedLocal + report.IDDelim + scopedRemote ) @@ -56,7 +56,7 @@ func addConnection( if _, ok := r.Network.NodeMetadatas[scopedLocal]; !ok { r.Network.NodeMetadatas[scopedLocal] = report.NodeMetadata{ - "name": nodeName, + "name": hostName, } } @@ -68,9 +68,9 @@ func addConnection( if c.Proc.PID > 0 { var ( - scopedLocal = scopedIPPort(nodeID, c.LocalAddress, c.LocalPort) - scopedRemote = scopedIPPort(nodeID, c.RemoteAddress, c.RemotePort) - key = nodeID + report.IDDelim + scopedLocal + scopedLocal = scopedIPPort(hostID, c.LocalAddress, c.LocalPort) + scopedRemote = scopedIPPort(hostID, c.RemoteAddress, c.RemotePort) + key = hostID + report.IDDelim + scopedLocal edgeKey = scopedLocal + report.IDDelim + scopedRemote ) @@ -81,7 +81,7 @@ func addConnection( md := report.NodeMetadata{ "pid": fmt.Sprintf("%d", c.Proc.PID), "name": c.Proc.Name, - "domain": nodeID, + "domain": hostID, } for _, pm := range pms { diff --git a/probe/spy_test.go b/probe/spy_test.go index d6f7239afb..a7e80df1c1 100644 --- a/probe/spy_test.go +++ b/probe/spy_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "net" "strconv" "testing" @@ -89,8 +88,8 @@ func TestSpyNetwork(t *testing.T) { ) r := spy(nodeID, nodeName, false, []processMapper{}) - buf, _ := json.MarshalIndent(r, "", " ") - t.Logf("\n%s\n", buf) + //buf, _ := json.MarshalIndent(r, "", " ") + //t.Logf("\n%s\n", buf) // No process nodes, please if want, have := 0, len(r.Process.Adjacency); want != have { @@ -173,6 +172,4 @@ func TestSpyProcessDataSource(t *testing.T) { if want, have := v, r.Process.NodeMetadatas[scopedLocal][k]; want != have { t.Fatalf("%s: want %q, have %q", k, want, have) } - - t.Logf("%s: %q OK", k, v) } diff --git a/report/mapping_functions.go b/report/mapping_functions.go index 3cbc93ce9d..2a206db09e 100644 --- a/report/mapping_functions.go +++ b/report/mapping_functions.go @@ -28,76 +28,30 @@ type MapFunc func(string, NodeMetadata, bool) (MappedNode, bool) // ProcessPID takes a node NodeMetadata from a Process topology, and returns a // representation with the ID based on the process PID and the labels based // on the process name. -func ProcessPID(id string, m NodeMetadata, grouped bool) (MappedNode, bool) { +func ProcessPID(_ string, m NodeMetadata, grouped bool) (MappedNode, bool) { var ( - domain = m["domain"] - pid = m["pid"] - name = m["name"] - minor = fmt.Sprintf("%s (%s)", domain, pid) + identifier = fmt.Sprintf("%s:%s:%s", "pid", m["domain"], m["pid"]) + minor = fmt.Sprintf("%s (%s)", m["domain"], m["pid"]) + show = m["pid"] != "" && m["name"] != "" ) if grouped { - domain = "" - minor = "" + identifier = m["name"] // flatten + minor = "" // nothing meaningful to put here? } return MappedNode{ - ID: fmt.Sprintf("pid:%s:%s", domain, pid), - Major: name, + ID: identifier, + Major: m["name"], Minor: minor, - Rank: pid, - }, pid != "" -} - -// ProcessCgroup takes a node NodeMetadata from a Process topology, augmented -// with cgroup fields, and returns a representation based on the cgroup. If -// the cgroup is not present, it falls back to process name. -func ProcessCgroup(id string, m NodeMetadata, grouped bool) (MappedNode, bool) { - var ( - domain = m["domain"] - cgroup = m["cgroup"] - ) - - if cgroup == "" { - cgroup = m["name"] - } - - if grouped { - domain = "" - } - - return MappedNode{ - ID: fmt.Sprintf("cgroup:%s:%s", domain, cgroup), - Major: cgroup, - Minor: domain, - Rank: cgroup, - }, cgroup != "" -} - -// ProcessName takes a node NodeMetadata from a Process topology, and returns -// a representation based on the process name. -func ProcessName(id string, m NodeMetadata, grouped bool) (MappedNode, bool) { - var ( - name = m["name"] - domain = m["domain"] - ) - - if grouped { - domain = "" - } - - return MappedNode{ - ID: fmt.Sprintf("proc:%s:%s", domain, name), - Major: name, - Minor: domain, - Rank: name, - }, name != "" + Rank: m["pid"], + }, show } // NetworkHostname takes a node NodeMetadata from a Network topology, and // returns a representation based on the hostname. Major label is the // hostname, the minor label is the domain, if any. -func NetworkHostname(id string, m NodeMetadata, _ bool) (MappedNode, bool) { +func NetworkHostname(_ string, m NodeMetadata, _ bool) (MappedNode, bool) { var ( name = m["name"] domain = "" @@ -108,8 +62,6 @@ func NetworkHostname(id string, m NodeMetadata, _ bool) (MappedNode, bool) { domain = parts[1] } - // Note: no grouped special case. - return MappedNode{ ID: fmt.Sprintf("host:%s", name), Major: parts[0], @@ -117,31 +69,3 @@ func NetworkHostname(id string, m NodeMetadata, _ bool) (MappedNode, bool) { Rank: parts[0], }, name != "" } - -// NetworkIP takes a node NodeMetadata from a Network topology, and returns a -// representation based on the (scoped) IP. Major label is the IP, the Minor -// label is the hostname. -func NetworkIP(id string, m NodeMetadata, _ bool) (MappedNode, bool) { - var ( - name = m["name"] - ip = strings.SplitN(id, ScopeDelim, 2)[1] - ) - - // Note: no grouped special case. - - return MappedNode{ - ID: fmt.Sprintf("addr:%s", id), - Major: ip, - Minor: name, - Rank: ip, - }, id != "" -} - -// MapFuncRegistry maps a string to a MapFunc. -var MapFuncRegistry = map[string]MapFunc{ - "processpid": ProcessPID, - "processcgroup": ProcessCgroup, - "processname": ProcessName, - "networkhostname": NetworkHostname, - "networkip": NetworkIP, -} diff --git a/report/mapping_test.go b/report/mapping_test.go index d72378c53a..bb3ed957e9 100644 --- a/report/mapping_test.go +++ b/report/mapping_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestMapping(t *testing.T) { +func TestUngroupedMapping(t *testing.T) { for i, c := range []struct { f MapFunc id string @@ -37,32 +37,6 @@ func TestMapping(t *testing.T) { wantMinor: "", wantRank: "localhost", }, - { - f: NetworkIP, - id: ScopeDelim + "1.2.3.4", - meta: NodeMetadata{ - "name": "my.host", - }, - wantOK: true, - wantID: "addr:" + ScopeDelim + "1.2.3.4", - wantMajor: "1.2.3.4", - wantMinor: "my.host", - wantRank: "1.2.3.4", - }, - { - f: ProcessName, - id: "not-used-alpha", - meta: NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "hosta", - }, - wantOK: true, - wantID: "proc:hosta:curl", - wantMajor: "curl", - wantMinor: "hosta", - wantRank: "curl", - }, { f: ProcessPID, id: "not-used-beta", @@ -77,65 +51,6 @@ func TestMapping(t *testing.T) { wantMinor: "hosta (42)", wantRank: "42", }, - { - f: ProcessCgroup, - id: "not-used-delta", - meta: NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "hosta", - "cgroup": "systemd", - }, - wantOK: true, - wantID: "cgroup:hosta:systemd", - wantMajor: "systemd", - wantMinor: "hosta", - wantRank: "systemd", - }, - { - f: ProcessCgroup, - id: "not-used-kappa", - meta: NodeMetadata{ - "pid": "42536", - "domain": "hosta", - "cgroup": "", // missing cgroup, and - "name": "", // missing name - }, - wantOK: false, - wantID: "cgroup:hosta:", - wantMajor: "", - wantMinor: "hosta", - wantRank: "", - }, - { - f: ProcessCgroup, - id: "not-used-gamma", - meta: NodeMetadata{ - "pid": "42536", - "domain": "hosta", - "cgroup": "", // missing cgroup, but - "name": "elasticsearch", // having name - }, - wantOK: true, - wantID: "cgroup:hosta:elasticsearch", - wantMajor: "elasticsearch", - wantMinor: "hosta", - wantRank: "elasticsearch", - }, - { - f: ProcessName, - id: "not-used-iota", - meta: NodeMetadata{ - "pid": "42", - "domain": "hosta", - "name": "", // missing name - }, - wantOK: false, - wantID: "proc:hosta:", - wantMajor: "", - wantMinor: "hosta", - wantRank: "", - }, } { identity := fmt.Sprintf("(%d %s %v)", i, c.id, c.meta) @@ -157,3 +72,7 @@ func TestMapping(t *testing.T) { } } } + +func TestGroupedMapping(t *testing.T) { + t.Skipf("not yet implemented") // TODO +} diff --git a/report/metadata.go b/report/metadata.go index 7ff9ee8065..7e5d3aa762 100644 --- a/report/metadata.go +++ b/report/metadata.go @@ -1,7 +1,7 @@ package report // AggregateMetadata is a composable version of an EdgeMetadata. It's used -// when we want to merge nodes/edges for any reason, or +// when we want to merge nodes/edges for any reason. // // Even though we base it on EdgeMetadata, we can apply it to nodes, by // summing up (merging) all of the {ingress, egress} metadatas of the diff --git a/report/report.go b/report/report.go index 5dc2d6c120..0863539203 100644 --- a/report/report.go +++ b/report/report.go @@ -41,14 +41,15 @@ type HostMetadata struct { // an element of a topology. It should contain information that's relevant // to rendering a node when there are many nodes visible at once. type RenderableNode struct { - ID string `json:"id"` // - LabelMajor string `json:"label_major"` // e.g. "process", human-readable - LabelMinor string `json:"label_minor,omitempty"` // e.g. "hostname", human-readable, optional - Rank string `json:"rank"` // to help with the layout engine - Pseudo bool `json:"pseudo,omitempty"` // sort-of a placeholder node, for rendering purposes - Adjacency IDList `json:"adjacency,omitempty"` // Node IDs - Origin IDList `json:"origin,omitempty"` // Origin IDs - Metadata AggregateMetadata `json:"metadata"` // sums + ID string `json:"id"` // + LabelMajor string `json:"label_major"` // e.g. "process", human-readable + LabelMinor string `json:"label_minor,omitempty"` // e.g. "hostname", human-readable, optional + Rank string `json:"rank"` // to help the layout engine + Pseudo bool `json:"pseudo,omitempty"` // sort-of a placeholder node, for rendering purposes + Adjacency IDList `json:"adjacency,omitempty"` // Node IDs (in the same topology domain) + OriginHosts IDList `json:"origin_hosts,omitempty"` // Which hosts contributed information to this node + OriginNodes IDList `json:"origin_nodes,omitempty"` // Which origin nodes (depends on topology) contributed + Metadata AggregateMetadata `json:"metadata"` // Numeric sums } // DetailedNode is the data type that's yielded to the JavaScript layer when diff --git a/report/topology.go b/report/topology.go index 51eca22d37..1c7e1bbe4c 100644 --- a/report/topology.go +++ b/report/topology.go @@ -87,12 +87,15 @@ func (t Topology) RenderBy(f MapFunc, grouped bool) map[string]RenderableNode { // the existing data, on the assumption that the MapFunc returns the same // data. nodes[mapped.ID] = RenderableNode{ - ID: mapped.ID, - LabelMajor: mapped.Major, - LabelMinor: mapped.Minor, - Rank: mapped.Rank, - Pseudo: false, - Metadata: AggregateMetadata{}, // can only fill in later + ID: mapped.ID, + LabelMajor: mapped.Major, + LabelMinor: mapped.Minor, + Rank: mapped.Rank, + Pseudo: false, + Adjacency: IDList{}, // later + OriginHosts: IDList{}, // later + OriginNodes: IDList{}, // later + Metadata: AggregateMetadata{}, // later } address2mapped[addressID] = mapped.ID } @@ -101,7 +104,7 @@ func (t Topology) RenderBy(f MapFunc, grouped bool) map[string]RenderableNode { for src, dsts := range t.Adjacency { var ( fields = strings.SplitN(src, IDDelim, 2) // "|
" - srcNodeID = fields[0] + srcOriginHostID = fields[0] srcNodeAddress = fields[1] srcRenderableID = address2mapped[srcNodeAddress] // must exist srcRenderableNode = nodes[srcRenderableID] // must exist @@ -133,8 +136,9 @@ func (t Topology) RenderBy(f MapFunc, grouped bool) map[string]RenderableNode { address2mapped[dstNodeAddress] = dstRenderableID } - srcRenderableNode.Origin = srcRenderableNode.Origin.Add(srcNodeID) srcRenderableNode.Adjacency = srcRenderableNode.Adjacency.Add(dstRenderableID) + srcRenderableNode.OriginHosts = srcRenderableNode.OriginHosts.Add(srcOriginHostID) + srcRenderableNode.OriginNodes = srcRenderableNode.OriginNodes.Add(srcNodeAddress) edgeID := srcNodeAddress + IDDelim + dstNodeAddress if md, ok := t.EdgeMetadatas[edgeID]; ok { srcRenderableNode.Metadata.Merge(md.Transform()) diff --git a/report/topology_test.go b/report/topology_test.go index 0e08dc0521..b0e0d35544 100644 --- a/report/topology_test.go +++ b/report/topology_test.go @@ -4,441 +4,251 @@ import ( "reflect" "sort" "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/pmezard/go-difflib/difflib" ) -var report = Report{ - Process: Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1;12345": NewIDList(";192.168.1.2;80"), - "hostA|;192.168.1.1;12346": NewIDList(";192.168.1.2;80"), - "hostA|;192.168.1.1;8888": NewIDList(";1.2.3.4;22"), - "hostB|;192.168.1.2;80": NewIDList(";192.168.1.1;12345"), - "hostB|;192.168.1.2;43201": NewIDList(";1.2.3.5;22"), - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1;12346|;192.168.1.2;80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1;8888|;1.2.3.4;22": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - }, - ";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.1.2;43201|;1.2.3.5;22": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 12, - }, - }, - NodeMetadatas: NodeMetadatas{ - ";192.168.1.1;12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - ";192.168.1.1;12346": NodeMetadata{ // <-- same as :12345 - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - ";192.168.1.1;8888": NodeMetadata{ - "pid": "55100", - "name": "ssh", - "domain": "node-a.local", - }, - ";192.168.1.2;80": NodeMetadata{ - "pid": "215", - "name": "apache", - "domain": "node-b.local", - }, - ";192.168.1.2;43201": NodeMetadata{ - "pid": "8765", - "name": "ssh", - "domain": "node-b.local", - }, - }, - }, +func init() { + spew.Config.SortKeys = true // :\ +} - Network: Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1": NewIDList(";192.168.1.2", ";1.2.3.4"), - "hostB|;192.168.1.2": NewIDList(";192.168.1.1", ";1.2.3.5"), - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1|;192.168.1.2": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1|;1.2.3.4": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - }, - ";192.168.1.2|;192.168.1.1": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.1.2|;1.2.3.5": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 12, - }, - }, - NodeMetadatas: NodeMetadatas{ - ";192.168.1.1": NodeMetadata{ - "name": "host-a", - }, - ";192.168.1.2": NodeMetadata{ - "name": "host-b", - }, - }, - }, +const ( + client54001 = ScopeDelim + "10.10.10.20" + ScopeDelim + "54001" // curl (1) + client54002 = ScopeDelim + "10.10.10.20" + ScopeDelim + "54002" // curl (2) + server80 = ScopeDelim + "192.168.1.1" + ScopeDelim + "80" // apache - HostMetadatas: HostMetadatas{ - "hostA": HostMetadata{ - Hostname: "node-a.local", - OS: "Linux", - }, - "hostB": HostMetadata{ - Hostname: "node-b.local", - OS: "Linux", - }, - }, -} + clientIP = ScopeDelim + "10.10.10.20" + serverIP = ScopeDelim + "192.168.1.1" + randomIP = ScopeDelim + "172.16.11.9" // only in Network topology +) -func TestTopologyProc(t *testing.T) { - // Process topology with by-processname mapping - { - if want, have := map[string]RenderableNode{ - "proc:node-b.local:apache": { - ID: "proc:node-b.local:apache", - LabelMajor: "apache", - LabelMinor: "node-b.local", - Rank: "apache", - Pseudo: false, - Adjacency: NewIDList("proc:node-a.local:curl"), - Origin: NewIDList("hostB"), - Metadata: AggregateMetadata{ - "egress_bytes": 0, - "ingress_bytes": 12, +var ( + report = Report{ + Process: Topology{ + Adjacency: Adjacency{ + "client.hostname.com" + IDDelim + client54001: NewIDList(server80), + "client.hostname.com" + IDDelim + client54002: NewIDList(server80), + "server.hostname.com" + IDDelim + server80: NewIDList(client54001, client54002), + }, + NodeMetadatas: NodeMetadatas{ + // NodeMetadata is arbitrary. We're free to put only precisely what we + // care to test into the fixture. Just be sure to include the bits + // that the mapping funcs extract :) + client54001: NodeMetadata{ + "name": "curl", + "domain": "client-54001-domain", + "pid": "10001", }, - }, - "proc:node-a.local:curl": { - ID: "proc:node-a.local:curl", - LabelMajor: "curl", - LabelMinor: "node-a.local", - Rank: "curl", - Pseudo: false, - Adjacency: NewIDList("proc:node-b.local:apache"), - Origin: NewIDList("hostA"), - Metadata: AggregateMetadata{ - "egress_bytes": 24, - "ingress_bytes": 0, + client54002: NodeMetadata{ + "name": "curl", // should be same as above! + "domain": "client-54002-domain", // may be different than above + "pid": "10001", // should be same as above! }, - }, - "proc:node-a.local:ssh": { - ID: "proc:node-a.local:ssh", - LabelMajor: "ssh", - LabelMinor: "node-a.local", - Rank: "ssh", - Pseudo: false, - Adjacency: NewIDList("pseudo:;1.2.3.4;22"), - Origin: NewIDList("hostA"), - Metadata: AggregateMetadata{ - "egress_bytes": 200, - "ingress_bytes": 0, + server80: NodeMetadata{ + "name": "apache", + "domain": "server-80-domain", + "pid": "215", }, }, - "proc:node-b.local:ssh": { - ID: "proc:node-b.local:ssh", - LabelMajor: "ssh", - LabelMinor: "node-b.local", - Rank: "ssh", - Pseudo: false, - Adjacency: NewIDList("pseudo:;1.2.3.5;22"), - Origin: NewIDList("hostB"), - Metadata: AggregateMetadata{ - "egress_bytes": 200, - "ingress_bytes": 12, + EdgeMetadatas: EdgeMetadatas{ + client54001 + IDDelim + server80: EdgeMetadata{ + WithBytes: true, + BytesIngress: 100, + BytesEgress: 10, + }, + client54002 + IDDelim + server80: EdgeMetadata{ + WithBytes: true, + BytesIngress: 200, + BytesEgress: 20, + }, + server80 + IDDelim + client54001: EdgeMetadata{ + WithBytes: true, + BytesIngress: 10, + BytesEgress: 100, + }, + server80 + IDDelim + client54002: EdgeMetadata{ + WithBytes: true, + BytesIngress: 20, + BytesEgress: 200, }, }, - "pseudo:;1.2.3.4;22": { - ID: "pseudo:;1.2.3.4;22", - LabelMajor: "1.2.3.4:22", - Pseudo: true, - Metadata: AggregateMetadata{}, + }, + Network: Topology{ + Adjacency: Adjacency{ + "client.hostname.com" + IDDelim + clientIP: NewIDList(serverIP), + "random.hostname.com" + IDDelim + randomIP: NewIDList(serverIP), + "server.hostname.com" + IDDelim + serverIP: NewIDList(clientIP), // no backlink to random + }, + NodeMetadatas: NodeMetadatas{ + clientIP: NodeMetadata{ + "name": "client.hostname.com", // hostname + }, + randomIP: NodeMetadata{ + "name": "random.hostname.com", // hostname + }, + serverIP: NodeMetadata{ + "name": "server.hostname.com", // hostname + }, }, - "pseudo:;1.2.3.5;22": { - ID: "pseudo:;1.2.3.5;22", - LabelMajor: "1.2.3.5:22", - Pseudo: true, - Metadata: AggregateMetadata{}, + EdgeMetadatas: EdgeMetadatas{ + clientIP + IDDelim + serverIP: EdgeMetadata{ + WithConnCountTCP: true, + MaxConnCountTCP: 3, + }, + randomIP + IDDelim + serverIP: EdgeMetadata{ + WithConnCountTCP: true, + MaxConnCountTCP: 20, // dangling connections, weird but possible + }, + serverIP + IDDelim + clientIP: EdgeMetadata{ + WithConnCountTCP: true, + MaxConnCountTCP: 3, + }, }, - }, report.Process.RenderBy(ProcessName, false); !reflect.DeepEqual(want, have) { - t.Errorf("want\n\t%#v, have\n\t%#v", want, have) - } - } - - // check EdgeMetadata - { - want := EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - } - have := report.Process.EdgeMetadata( - ProcessName, - false, - "proc:node-b.local:apache", - "proc:node-a.local:curl", - ) - if want != have { - t.Errorf("Topology error. Want:\n%#v\nHave:\n%#v\n", want, have) - } + }, } -} +) -func TestTopologyProcClass(t *testing.T) { - // Process name classes. - { - if want, have := map[string]RenderableNode{ - "proc::apache": { - ID: "proc::apache", - LabelMajor: "apache", - LabelMinor: "", - Rank: "apache", - Pseudo: false, - Adjacency: NewIDList("proc::curl"), - Origin: NewIDList("hostB"), - Metadata: AggregateMetadata{ - "egress_bytes": 0, - "ingress_bytes": 12, - }, +func TestRenderByProcessPID(t *testing.T) { + want := map[string]RenderableNode{ + "pid:client-54001-domain:10001": { + ID: "pid:client-54001-domain:10001", + LabelMajor: "curl", + LabelMinor: "client-54001-domain (10001)", + Rank: "10001", + Pseudo: false, + Adjacency: NewIDList("pid:server-80-domain:215"), + OriginHosts: NewIDList("client.hostname.com"), + OriginNodes: NewIDList(";10.10.10.20;54001"), + Metadata: AggregateMetadata{ + KeyBytesIngress: 100, + KeyBytesEgress: 10, }, - "proc::curl": { - ID: "proc::curl", - LabelMajor: "curl", - LabelMinor: "", - Rank: "curl", - Pseudo: false, - Adjacency: NewIDList("proc::apache"), - Origin: NewIDList("hostA"), - Metadata: AggregateMetadata{ - "egress_bytes": 24, - "ingress_bytes": 0, - }, - }, - "proc::ssh": { - ID: "proc::ssh", - LabelMajor: "ssh", - LabelMinor: "", - Rank: "ssh", - Pseudo: false, - Adjacency: NewIDList(localUnknown), - Origin: NewIDList("hostA", "hostB"), - Metadata: AggregateMetadata{ - "egress_bytes": 400, - "ingress_bytes": 12, - }, + }, + "pid:client-54002-domain:10001": { + ID: "pid:client-54002-domain:10001", + LabelMajor: "curl", + LabelMinor: "client-54002-domain (10001)", + Rank: "10001", // same process + Pseudo: false, + Adjacency: NewIDList("pid:server-80-domain:215"), + OriginHosts: NewIDList("client.hostname.com"), + OriginNodes: NewIDList(";10.10.10.20;54002"), + Metadata: AggregateMetadata{ + KeyBytesIngress: 200, + KeyBytesEgress: 20, }, - localUnknown: { - ID: localUnknown, - LabelMajor: "", - LabelMinor: "", - Pseudo: true, - Metadata: AggregateMetadata{}, + }, + "pid:server-80-domain:215": { + ID: "pid:server-80-domain:215", + LabelMajor: "apache", + LabelMinor: "server-80-domain (215)", + Rank: "215", + Pseudo: false, + Adjacency: NewIDList("pid:client-54001-domain:10001", "pid:client-54002-domain:10001"), + OriginHosts: NewIDList("server.hostname.com"), + OriginNodes: NewIDList(";192.168.1.1;80"), + Metadata: AggregateMetadata{ + KeyBytesIngress: 30, + KeyBytesEgress: 300, }, - }, report.Process.RenderBy(ProcessName, true); !reflect.DeepEqual(want, have) { - t.Errorf("want\n\t%#v, have\n\t%#v", want, have) - } + }, } - - // check EdgeMetadata - { - want := EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - } - have := report.Process.EdgeMetadata( - ProcessName, - true, // class view - "proc::apache", - "proc::curl", - ) - if want != have { - t.Errorf("Topology error. Want:\n%#v\nHave:\n%#v\n", want, have) - } + have := report.Process.RenderBy(ProcessPID, false) + if !reflect.DeepEqual(want, have) { + t.Error("\n" + diff(want, have)) } } -func TestTopologyHost(t *testing.T) { - // Network topology with by-hostname mapping - { - want := map[string]RenderableNode{ - "host:host-a": { - ID: "host:host-a", - LabelMajor: "host-a", - Rank: "host-a", - Pseudo: false, - Adjacency: NewIDList( - "pseudo:;1.2.3.4", - "host:host-b", - ), - Origin: NewIDList("hostA"), - Metadata: AggregateMetadata{ - "egress_bytes": 212, - "ingress_bytes": 0, - }, +func TestRenderByProcessPIDGrouped(t *testing.T) { + // For grouped, I've somewhat arbitrarily chosen to squash together all + // processes with the same name by removing the PID and domain (host) + // dimensions from the ID. That could be changed. + want := map[string]RenderableNode{ + "curl": { + ID: "curl", + LabelMajor: "curl", + LabelMinor: "", + Rank: "10001", + Pseudo: false, + Adjacency: NewIDList("apache"), + OriginHosts: NewIDList("client.hostname.com"), + OriginNodes: NewIDList(";10.10.10.20;54001", ";10.10.10.20;54002"), + Metadata: AggregateMetadata{ + KeyBytesIngress: 300, + KeyBytesEgress: 30, }, - "host:host-b": { - ID: "host:host-b", - LabelMajor: "host-b", - Rank: "host-b", - Pseudo: false, - Adjacency: NewIDList( - "host:host-a", - "pseudo:;1.2.3.5", - ), - Origin: NewIDList("hostB"), - Metadata: AggregateMetadata{ - "egress_bytes": 200, - "ingress_bytes": 24, - }, - }, - "pseudo:;1.2.3.4": { - ID: "pseudo:;1.2.3.4", - LabelMajor: "1.2.3.4", - Pseudo: true, - Metadata: AggregateMetadata{}, - }, - "pseudo:;1.2.3.5": { - ID: "pseudo:;1.2.3.5", - LabelMajor: "1.2.3.5", - Pseudo: true, - Metadata: AggregateMetadata{}, + }, + "apache": { + ID: "apache", + LabelMajor: "apache", + LabelMinor: "", + Rank: "215", + Pseudo: false, + Adjacency: NewIDList("curl"), + OriginHosts: NewIDList("server.hostname.com"), + OriginNodes: NewIDList(";192.168.1.1;80"), + Metadata: AggregateMetadata{ + KeyBytesIngress: 30, + KeyBytesEgress: 300, }, - } - - have := report.Network.RenderBy(NetworkHostname, false) - - sort.Strings(have["net:host-a"].Adjacency) - - if !reflect.DeepEqual(want, have) { - t.Errorf("want\n\t%#v, have\n\t%#v", want, have) - } + }, } - - // check EdgeMetadata - { - want := EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - } - have := report.Network.EdgeMetadata( - NetworkHostname, - false, - "host:host-b", - "host:host-a", - ) - if want != have { - t.Errorf("Topology error. Want:\n%#v\nHave:\n%#v\n", want, have) - } + have := report.Process.RenderBy(ProcessPID, true) + if !reflect.DeepEqual(want, have) { + t.Error("\n" + diff(want, have)) } } -func TestTopologyIP(t *testing.T) { - // Network topology with by-IP mapping - { - want := map[string]RenderableNode{ - "addr:;192.168.1.1": { - ID: "addr:;192.168.1.1", - LabelMajor: "192.168.1.1", - LabelMinor: "host-a", - Rank: "192.168.1.1", - Pseudo: false, - Adjacency: NewIDList( - "pseudo:;1.2.3.4", - "addr:;192.168.1.2", - ), - Origin: NewIDList("hostA"), - Metadata: AggregateMetadata{ - "egress_bytes": 212, - "ingress_bytes": 0, - }, - }, - "addr:;192.168.1.2": { - ID: "addr:;192.168.1.2", - LabelMajor: "192.168.1.2", - LabelMinor: "host-b", - Rank: "192.168.1.2", - Pseudo: false, - Adjacency: NewIDList( - "pseudo:;1.2.3.5", - "addr:;192.168.1.1", - ), - Origin: NewIDList("hostB"), - Metadata: AggregateMetadata{ - "egress_bytes": 200, - "ingress_bytes": 24, - }, +func TestRenderByNetworkHostname(t *testing.T) { + want := map[string]RenderableNode{ + "host:client.hostname.com": { + ID: "host:client.hostname.com", + LabelMajor: "client", // before first . + LabelMinor: "hostname.com", // after first . + Rank: "client", + Pseudo: false, + Adjacency: NewIDList("host:server.hostname.com"), + OriginHosts: NewIDList("client.hostname.com"), + OriginNodes: NewIDList(";10.10.10.20"), + Metadata: AggregateMetadata{ + KeyMaxConnCountTCP: 3, }, - "pseudo:;1.2.3.4": { - ID: "pseudo:;1.2.3.4", - LabelMajor: "1.2.3.4", - Pseudo: true, - Metadata: AggregateMetadata{}, + }, + "host:random.hostname.com": { + ID: "host:random.hostname.com", + LabelMajor: "random", // before first . + LabelMinor: "hostname.com", // after first . + Rank: "random", + Pseudo: false, + Adjacency: NewIDList("host:server.hostname.com"), + OriginHosts: NewIDList("random.hostname.com"), + OriginNodes: NewIDList(";172.16.11.9"), + Metadata: AggregateMetadata{ + KeyMaxConnCountTCP: 20, }, - "pseudo:;1.2.3.5": { - ID: "pseudo:;1.2.3.5", - LabelMajor: "1.2.3.5", - Pseudo: true, - Metadata: AggregateMetadata{}, + }, + "host:server.hostname.com": { + ID: "host:server.hostname.com", + LabelMajor: "server", // before first . + LabelMinor: "hostname.com", // after first . + Rank: "server", + Pseudo: false, + Adjacency: NewIDList("host:client.hostname.com"), + OriginHosts: NewIDList("server.hostname.com"), + OriginNodes: NewIDList(";192.168.1.1"), + Metadata: AggregateMetadata{ + KeyMaxConnCountTCP: 3, }, - } - have := report.Network.RenderBy(NetworkIP, false) - sort.Strings(have["pseudo:;192.168.1.1"].Adjacency) - if !reflect.DeepEqual(want, have) { - t.Errorf("want\n\t%#v, have\n\t%#v", want, have) - } + }, } - - // check EdgeMetadata - { - want := EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - } - have := report.Network.EdgeMetadata( - NetworkIP, - false, - "addr:;192.168.1.1", - "addr:;192.168.1.2", - ) - if want != have { - t.Errorf("Topology error. Want:\n%#v\nHave:\n%#v\n", want, have) - } + have := report.Network.RenderBy(NetworkHostname, true) + if !reflect.DeepEqual(want, have) { + t.Error("\n" + diff(want, have)) } } -func TestTopologyDiff(t *testing.T) { - // Diff renderable nodes. +func TestTopoDiff(t *testing.T) { nodea := RenderableNode{ ID: "nodea", LabelMajor: "Node A", @@ -451,7 +261,7 @@ func TestTopologyDiff(t *testing.T) { nodeap := nodea nodeap.Adjacency = []string{ "nodeb", - "nodeq", // not the same anymore. + "nodeq", // not the same anymore } nodeb := RenderableNode{ ID: "nodeb", @@ -467,39 +277,40 @@ func TestTopologyDiff(t *testing.T) { return r } - for i, c := range []struct { + for _, c := range []struct { + label string have, want Diff }{ { - // basecase: empty -> something - have: TopoDiff(nodes(), nodes(nodea, nodeb)), + label: "basecase: empty -> something", + have: TopoDiff(nodes(), nodes(nodea, nodeb)), want: Diff{ Add: []RenderableNode{nodea, nodeb}, }, }, { - // basecase: something -> empty - have: TopoDiff(nodes(nodea, nodeb), nodes()), + label: "basecase: something -> empty", + have: TopoDiff(nodes(nodea, nodeb), nodes()), want: Diff{ Remove: []string{"nodea", "nodeb"}, }, }, { - // add and remove - have: TopoDiff(nodes(nodea), nodes(nodeb)), + label: "add and remove", + have: TopoDiff(nodes(nodea), nodes(nodeb)), want: Diff{ Add: []RenderableNode{nodeb}, Remove: []string{"nodea"}, }, }, { - // no change. - have: TopoDiff(nodes(nodea), nodes(nodea)), - want: Diff{}, + label: "no change", + have: TopoDiff(nodes(nodea), nodes(nodea)), + want: Diff{}, }, { - // change a single node - have: TopoDiff(nodes(nodea), nodes(nodeap)), + label: "change a single node", + have: TopoDiff(nodes(nodea), nodes(nodeap)), want: Diff{ Update: []RenderableNode{nodeap}, }, @@ -509,7 +320,18 @@ func TestTopologyDiff(t *testing.T) { sort.Sort(ByID(c.have.Add)) sort.Sort(ByID(c.have.Update)) if !reflect.DeepEqual(c.want, c.have) { - t.Errorf("case %d: want\n\t%#v, have\n\t%#v", i, c.want, c.have) + t.Errorf("%s\n%s", c.label, diff(c.want, c.have)) } } } + +func diff(want, have interface{}) string { + text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(spew.Sdump(want)), + B: difflib.SplitLines(spew.Sdump(have)), + FromFile: "want", + ToFile: "have", + Context: 3, + }) + return text +}