Skip to content

Commit

Permalink
Feature/release 1.10.0 (#72)
Browse files Browse the repository at this point in the history
* + PlantUML

* + graph compression

* + minor fixes and format improvements
  • Loading branch information
s0rg authored Mar 19, 2024
1 parent 3386c5d commit 6b06f1a
Show file tree
Hide file tree
Showing 49 changed files with 1,791 additions and 383 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
dist/
vendor/
.structurizr/
workspace.*
/*.json
*.out
*.png
*.svg
Expand Down
22 changes: 17 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ Takes all network connections from your docker containers and exports them as:
- [graphviz dot](https://www.graphviz.org/doc/info/lang.html)
- [structurizr dsl](https://github.com/structurizr/dsl)
- [compose yaml](https://github.com/compose-spec/compose-spec/blob/master/spec.md)
- [plant uml](https://github.com/plantuml/plantuml)
- pseudographical tree
- json stream
- statistics - nodes, connections and listen ports counts
Expand Down Expand Up @@ -53,6 +54,7 @@ Closest analogs, i can find, that not suit my needs very well:
- all output formats are sorted, thus can be placed to any `vcs` to observe changes
- fast, scans ~470 containers with ~4000 connections in around 5 sec
- auto-clusterization based on graph topology
- deep inspection mode, in wich connections between procesess inside containers, also collected and shown
- 100% test-coverage

## known limitations
Expand All @@ -74,12 +76,14 @@ decompose [flags]
-cluster string
json file with clusterization rules, or auto:<similarity> for auto-clustering, similarity is float in (0.0, 1.0] range
-compress
compress graph
-deep
process-based introspection
-follow string
follow only this container by name(s), comma-separated or from @file
-format string
output format: json, csv, dot, yaml, stat, tree or sdsl for structurizr dsl (default "json")
output format: csv, dot, json, puml, sdsl, stat, tree, yaml (default "json")
-full
extract full process info: (cmd, args, env) and volumes info
-help
Expand Down Expand Up @@ -122,8 +126,12 @@ type Item struct {
Cmd []string `json:"cmd"`
Env []string `json:"env"`
Labels map[string]string `json:"labels"`
} `json:"container"` // conatiner info
Listen map[string][]string `json:"listen"` // ports with process names
} `json:"container"` // container info
Listen map[string][]{
Kind string `json:"kind"` // tcp / udp
Value int `json:"value"`
Local bool `json:"local"` // bound to loopback
} `json:"listen"` // ports with process names
Networks []string `json:"networks"` // network names
Tags []string `json:"tags"` // tags, if meta presents
Volumes []*struct{
Expand Down Expand Up @@ -152,7 +160,9 @@ Single node example with full info and metadata filled:
],
"labels": {}
},
"listen": {"foo": ["80/tcp"]},
"listen": {"foo": [
{"kind": "tcp", "value": 80}
]},
"networks": ["test-net"],
"tags": ["some"],
"volumes": [
Expand All @@ -168,7 +178,9 @@ Single node example with full info and metadata filled:
}
],
"connected": {
"bar-1": ["443/tcp"]
"bar-1": [
{"src": "foo", "dst": "[remote]", "port": {"kind": "tcp", "value": 443}}
]
}
}
```
Expand Down
4 changes: 2 additions & 2 deletions SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@

| Version | Supported |
| ------- | ------------------ |
| 1.9.x | :white_check_mark: |
| \< 1.9 | :x: |
| 1.10.x | :white_check_mark: |
| \< 1.10 | :x: |

## Reporting a Vulnerability

Expand Down
27 changes: 18 additions & 9 deletions cmd/decompose/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"os"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"

Expand Down Expand Up @@ -44,14 +45,15 @@ var (
fSilent, fVersion bool
fHelp, fLocal bool
fFull, fNoLoops bool
fDeep bool
fDeep, fCompress bool
fProto, fFormat string
fOut, fFollow string
fMeta, fCluster string
fSkipEnv string
fLoad []string

ErrUnknown = errors.New("unknown")
knownBuilders string
ErrUnknown = errors.New("unknown")
)

func version() string {
Expand Down Expand Up @@ -85,6 +87,8 @@ func setupFlags() {
flag.BoolVar(&fFull, "full", false, "extract full process info: (cmd, args, env) and volumes info")
flag.BoolVar(&fNoLoops, "no-loops", false, "remove connection loops (node to itself) from output")
flag.BoolVar(&fDeep, "deep", false, "process-based introspection")
flag.BoolVar(&fCompress, "compress", false, "compress graph")

flag.StringVar(&fOut, "out", defaultOutput, "output: filename or \"-\" for stdout")
flag.StringVar(&fMeta, "meta", "", "json file with metadata for enrichment")
flag.StringVar(&fProto, "proto", defaultProto, "protocol to scan: tcp, udp or all")
Expand All @@ -97,12 +101,8 @@ func setupFlags() {
"similarity is float in (0.0, 1.0] range",
)

flag.StringVar(
&fFormat,
"format",
builder.KindJSON,
"output format: json, csv, dot, yaml, stat, tree or sdsl for structurizr dsl",
)
flag.StringVar(&fFormat, "format", builder.KindJSON, "output format: "+knownBuilders)

flag.StringVar(
&fSkipEnv,
"skip-env",
Expand Down Expand Up @@ -262,7 +262,7 @@ func prepareConfig() (
"%w format: %s known: %s",
ErrUnknown,
fFormat,
strings.Join(builder.Names(), ","),
knownBuilders,
)
}

Expand Down Expand Up @@ -290,6 +290,12 @@ func prepareConfig() (
bildr, nwr = cb, cb
}

if fCompress {
cmp := graph.NewCompressor(bildr)

bildr, nwr = cmp, cmp
}

skipKeys := []string{}

if fSkipEnv != "" {
Expand Down Expand Up @@ -396,6 +402,9 @@ func doBuild(
}

func main() {
slices.Sort(builder.Names)
knownBuilders = strings.Join(builder.Names, ", ")

setupFlags()

flag.Parse()
Expand Down
47 changes: 26 additions & 21 deletions examples/stream.json
Original file line number Diff line number Diff line change
@@ -1,40 +1,45 @@
{
"name": "nginx-1",
"is_external": false,
"listen": {"nginx": ["80/tcp"]},
"name": "nginx1",
"listen": {"nginx": [{"kind": "tcp", "value": 80}]},
"connected": {
"back-1": [{"src": "nginx", "dst": "app", "port": "8080/tcp"}],
"back-2": [{"src": "nginx", "dst": "app", "port": "8081/tcp"}]
"back1": [{"src": "nginx", "dst": "app", "port": {"kind": "tcp", "value": 8080}}],
"back2": [{"src": "nginx", "dst": "app", "port": {"kind": "tcp", "value": 8081}}]
}
}
{
"name": "db-1",
"is_external": false,
"listen": {"postgres": ["5432/tcp"]},
"name": "db1",
"listen": {"postgres": [{"kind": "tcp", "value": 5432}]},
"connected": {}
}
{
"name": "back-1",
"is_external": false,
"listen": {"app": ["8080/tcp", "8081/tcp", "9000/tcp"]},
"name": "back1",
"listen": {"app": [
{"kind": "tcp", "value": 8080},
{"kind": "tcp", "value": 8081},
{"kind": "tcp", "value": 9000}
]},
"connected": {
"db-1": [{"src": "app", "dst": "postgres", "port": "5432/tcp"}]
"db1": [{"src": "app", "dst": "postgres", "port": {"kind": "tcp", "value": 5432}}]
}
}
{
"name": "back-2",
"is_external": false,
"listen": {"app": ["8080/tcp", "8081/tcp"]},
"name": "back2",
"listen": {"app": [
{"kind": "tcp", "value": 8080},
{"kind": "tcp", "value": 8081}
]},
"connected": {
"db-1": [{"src": "app", "dst": "postgres", "port": "5432/tcp"}],
"foo-1": [{"src": "app", "dst": "[remote]", "port": "9500/tcp"}]
"db1": [{"src": "app", "dst": "postgres", "port": {"kind": "tcp", "value": 5432}}],
"foo1": [{"src": "app", "dst": "[remote]", "port": {"kind": "tcp", "value": 9500}}]
}
}
{
"name": "foo-1",
"is_external": false,
"listen": {"[remote]": ["9500/tcp"]},
"name": "foo1",
"is_external": true,
"listen": {"[remote]": [{"kind": "tcp", "value": 9500}]},
"connected": {
"back-1": [{"src": "[remote]", "dst": "app", "port": "9000/tcp"}]
"back1": [
{"src": "[remote]", "dst": "app", "port": {"kind": "tcp", "value": 9000}}
]
}
}
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ module github.com/s0rg/decompose
go 1.22

require (
github.com/docker/docker v25.0.3+incompatible
github.com/docker/docker v25.0.4+incompatible
github.com/emicklei/dot v1.6.1
github.com/expr-lang/expr v1.16.1
github.com/prometheus/procfs v0.12.0
github.com/prometheus/procfs v0.13.0
github.com/s0rg/set v1.2.0
github.com/s0rg/trie v1.3.0
gopkg.in/yaml.v3 v3.0.1
Expand Down Expand Up @@ -34,10 +34,10 @@ require (
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.18.0 // indirect
golang.org/x/tools v0.19.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gotest.tools/v3 v3.5.0 // indirect
)
24 changes: 12 additions & 12 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v25.0.3+incompatible h1:D5fy/lYmY7bvZa0XTZ5/UJPljor41F+vdyJG5luQLfQ=
github.com/docker/docker v25.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v25.0.4+incompatible h1:XITZTrq+52tZyZxUOtFIahUf3aH367FLxJzt9vZeAF8=
github.com/docker/docker v25.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
Expand Down Expand Up @@ -58,8 +58,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/s0rg/set v1.2.0 h1:53b207YMktNQJXYei/oHuTR5oOO2e9+eieZOncYsh9g=
Expand Down Expand Up @@ -93,14 +93,14 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand All @@ -109,8 +109,8 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
Expand All @@ -121,8 +121,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
28 changes: 15 additions & 13 deletions internal/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,20 @@ const (
KindYAML = "yaml"
KindSTAT = "stat"
KindStructurizr = "sdsl"
KindPlantUML = "puml"
)

var Names = []string{
KindCSV,
KindDOT,
KindJSON,
KindTREE,
KindYAML,
KindSTAT,
KindStructurizr,
KindPlantUML,
}

func Create(kind string) (b graph.NamedBuilderWriter, ok bool) {
switch kind {
case KindCSV:
Expand All @@ -30,28 +42,18 @@ func Create(kind string) (b graph.NamedBuilderWriter, ok bool) {
return NewYAML(), true
case KindSTAT:
return NewStat(), true
case KindPlantUML:
return NewPlantUML(), true
}

return
}

func SupportCluster(n string) (yes bool) {
switch n {
case KindDOT, KindStructurizr, KindSTAT:
case KindDOT, KindStructurizr, KindSTAT, KindPlantUML:
return true
}

return false
}

func Names() (rv []string) {
return []string{
KindCSV,
KindDOT,
KindJSON,
KindTREE,
KindYAML,
KindSTAT,
KindStructurizr,
}
}
Loading

0 comments on commit 6b06f1a

Please sign in to comment.