Config file support

.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
 
     steps:
 
-      - name: set up go 1.18
+      - name: set up go 1.19
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
         id: go
 
       - name: checkout
@@ -40,7 +40,7 @@ jobs:
 
       - name: install golangci-lint and goveralls
         run: |
-          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.46.2
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.50.1
           GO111MODULE=off go get -u github.com/mattn/goveralls
 
       - name: run linters

.golangci.yml

@@ -42,11 +42,9 @@ linters:
     - gocyclo
     - dupl
     - misspell
-    - varcheck
-    - deadcode
+    - unused
     - typecheck
     - ineffassign
-    - varcheck
     - stylecheck
     - gochecknoinits
     - exportloopref
@@ -59,6 +57,9 @@ linters:
 
 issues:
   exclude-rules:
+    - text: "package-comments: should have a package comment"
+      linters:
+        - revive
     - text: "at least one file in a package should have a package comment"
       linters:
         - stylecheck

README.md

@@ -19,6 +19,7 @@ The idea of external services is to be able to integrate status of all related s
 
 ```
 Application Options:
+  -f, --config=      config file [$CONFIG]
   -l, --listen= listen on host:port (default: localhost:8080) [$LISTEN]
   -v, --volume= volumes to report (default: root:/) [$VOLUMES]
   -s, --service= services to report [$SERVICES]  
@@ -36,6 +37,40 @@ Help Options:
 * services (`--service`, can be repeated) is a list of name:url pairs, where name is a name of the service, and url is a url to the service. Supports `http`, `https`, `mongodb` and `docker` schemes. The response for each service will be in `services` field.
 * concurrency (`--concurrency`) is a number of concurrent requests to services.
 * timeout (`--timeout`) is a timeout for each request to services.
+* config file (`--config`, `-f`) is a path to the config file, see below for details.
+
+## configuration file 
+
+`sys-agent` can be configured with a yaml file as well. The file should contain a list of volumes and services. The file can be specified via `--config` or `-f` options or `CONFIG` environment variable. 
+
+```yml
+volumes:
+  - {name: root, path: /hostroot}
+  - {name: data, path: /data}
+
+services:
+  mongo:
+    - {name: dev, url: mongodb://example.com:27017, oplog_max_delta: 30m}
+  certificate:
+    - {name: prim_cert, url: https://example1.com}
+    - {name: second_cert, url: https://example2.com}
+  docker:
+    - {name: docker1, url: unix:///var/run/docker.sock, containers: [reproxy, mattermost, postgres]}
+    - {name: docker2, url: tcp://192.168.1.1:4080}
+  file:
+    - {name: first, path: /tmp/example1.txt}
+    - {name: second, path: /tmp/example2.txt}
+  http:
+    - {name: first, url: https://example1.com}
+    - {name: second, url: https://example2.com}
+  program:
+    - {name: first, path: /usr/bin/example1, args: [arg1, arg2]}
+    - {name: second, path: /usr/bin/example2}
+  nginx:
+    - {name: nginx, status_url: http://example.com:80}
+```
+
+The config file has the same structure as command line options. `sys-agent` converts the config file to command line options and then parses them as usual. 
 
 ## basic checks
 

app/config/config.go

@@ -0,0 +1,160 @@
+// Package config provides the configuration for the application.
+package config
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Parameters represents the whole configuration parameters
+type Parameters struct {
+	Volumes  []Volume `yaml:"volumes"`
+	Services struct {
+		HTTP        []HTTP        `yaml:"http"`
+		Certificate []Certificate `yaml:"certificate"`
+		File        []File        `yaml:"file"`
+		Mongo       []Mongo       `yaml:"mongo"`
+		Nginx       []Nginx       `yaml:"nginx"`
+		Program     []Program     `yaml:"program"`
+		Docker      []Docker      `yaml:"docker"`
+	} `yaml:"services"`
+
+	fileName string `yaml:"-"`
+}
+
+// Volume represents a volumes to check
+type Volume struct {
+	Name string `yaml:"name"`
+	Path string `yaml:"path"`
+}
+
+// HTTP represents a http service to check
+type HTTP struct {
+	Name string `yaml:"name"`
+	URL  string `yaml:"url"`
+}
+
+// Certificate represents a certificate to check
+type Certificate struct {
+	Name string `yaml:"name"`
+	URL  string `yaml:"url"`
+}
+
+// Docker represents a docker container to check
+type Docker struct {
+	Name       string   `yaml:"name"`
+	URL        string   `yaml:"url"`
+	Containers []string `yaml:"containers"` // required containers
+}
+
+// File represents a file to check
+type File struct {
+	Name string `yaml:"name"`
+	Path string `yaml:"path"`
+}
+
+// Mongo represents a mongo service to check
+type Mongo struct {
+	Name          string        `yaml:"name"`
+	URL           string        `yaml:"url"`
+	OplogMaxDelta time.Duration `yaml:"oplog_max_delta"`
+}
+
+// Nginx represents a nginx service to check
+type Nginx struct {
+	Name      string `yaml:"name"`
+	StatusURL string `yaml:"status_url"`
+}
+
+// Program represents a program to check
+type Program struct {
+	Name string   `yaml:"name"`
+	Path string   `yaml:"path"`
+	Args []string `yaml:"args"`
+}
+
+// New creates a new Parameters from the given file
+func New(fname string) (*Parameters, error) {
+	p := &Parameters{fileName: fname}
+	data, err := os.ReadFile(fname) // nolint gosec
+	if err != nil {
+		return nil, fmt.Errorf("can't read config %s: %w", fname, err)
+	}
+	if err = yaml.Unmarshal(data, &p); err != nil {
+		return nil, fmt.Errorf("failed to parsse config %s: %w", fname, err)
+	}
+	return p, nil
+}
+
+// MarshalVolumes returns the volumes as a list of strings with the format "name:path"
+func (p *Parameters) MarshalVolumes() []string {
+	res := make([]string, 0, len(p.Volumes))
+	for _, v := range p.Volumes {
+		res = append(res, fmt.Sprintf("%s:%s", v.Name, v.Path))
+	}
+	return res
+}
+
+// MarshalServices returns the services as a list of strings with the format used by command line
+func (p *Parameters) MarshalServices() []string {
+	res := []string{}
+
+	for _, v := range p.Services.HTTP {
+		res = append(res, fmt.Sprintf("%s:%s", v.Name, v.URL))
+	}
+
+	for _, v := range p.Services.Certificate {
+		url := strings.TrimPrefix(v.URL, "https://")
+		url = strings.TrimPrefix(url, "http://")
+		res = append(res, fmt.Sprintf("%s:cert://%s", v.Name, url))
+	}
+
+	for _, v := range p.Services.Docker {
+		url := strings.TrimPrefix(v.URL, "https://")
+		url = strings.TrimPrefix(url, "http://")
+		url = strings.TrimPrefix(url, "tcp://")
+		url = strings.TrimPrefix(url, "unix://")
+		if len(v.Containers) > 0 {
+			url += "?containers=" + strings.Join(v.Containers, ",")
+		}
+		res = append(res, fmt.Sprintf("%s:docker://%s", v.Name, url))
+	}
+
+	for _, v := range p.Services.File {
+		res = append(res, fmt.Sprintf("%s:file://%s", v.Name, v.Path))
+	}
+
+	for _, v := range p.Services.Mongo {
+		m := fmt.Sprintf("%s:%s", v.Name, v.URL)
+		if v.OplogMaxDelta > 0 {
+			if strings.Contains(m, "?") {
+				m += fmt.Sprintf("&oplogMaxDelta=%v", v.OplogMaxDelta)
+			} else {
+				m += fmt.Sprintf("?oplogMaxDelta=%v", v.OplogMaxDelta)
+			}
+		}
+		res = append(res, m)
+	}
+
+	for _, v := range p.Services.Nginx {
+		res = append(res, fmt.Sprintf("%s:nginx:%s", v.Name, v.StatusURL))
+	}
+
+	for _, v := range p.Services.Program {
+		prg := fmt.Sprintf("%s:program://%s", v.Name, v.Path)
+		if len(v.Args) > 0 {
+			prg += "?args=\"" + strings.Join(v.Args, " ") + "\""
+		}
+		res = append(res, prg)
+	}
+
+	return res
+}
+
+func (p *Parameters) String() string {
+	return fmt.Sprintf("config file: %q, %+v", p.fileName, *p)
+}

app/config/config_test.go

@@ -0,0 +1,81 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNew(t *testing.T) {
+	{
+		_, err := New("testdata/invalid.yml")
+		require.Error(t, err)
+		assert.EqualErrorf(t, err, "can't read config testdata/invalid.yml: open testdata/invalid.yml: no such file or directory", "expected error")
+	}
+
+	{
+		p, err := New("testdata/config.yml")
+		require.NoError(t, err)
+		assert.Equal(t, []Volume{{Name: "root", Path: "/hostroot"}, {Name: "data", Path: "/data"}}, p.Volumes)
+		assert.Equal(t, []Certificate{{Name: "prim_cert", URL: "https://example1.com"},
+			{Name: "second_cert", URL: "https://example2.com"}}, p.Services.Certificate)
+		assert.Equal(t, []Docker{
+			{Name: "docker1", URL: "unix:///var/run/docker.sock", Containers: []string{"reproxy", "mattermost", "postgres"}},
+			{Name: "docker2", URL: "tcp://192.168.1.1:4080", Containers: []string(nil)}}, p.Services.Docker)
+		assert.Equal(t, []File{{Name: "first", Path: "/tmp/example1.txt"}, {Name: "second", Path: "/tmp/example2.txt"}},
+			p.Services.File)
+		assert.Equal(t, []HTTP{{Name: "first", URL: "https://example1.com"}, {Name: "second", URL: "https://example2.com"}},
+			p.Services.HTTP)
+		assert.Equal(t, []Mongo{{Name: "dev", URL: "mongodb://example.com:27017", OplogMaxDelta: 30 * time.Minute}},
+			p.Services.Mongo)
+		assert.Equal(t, []Nginx{{Name: "nginx", StatusURL: "http://example.com:80"}}, p.Services.Nginx)
+	}
+}
+
+func TestParameters_MarshalVolumes(t *testing.T) {
+	p, err := New("testdata/config.yml")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"root:/hostroot", "data:/data"}, p.MarshalVolumes())
+}
+
+func TestParameters_String(t *testing.T) {
+	p, err := New("testdata/config.yml")
+	require.NoError(t, err)
+	exp := `config file: "testdata/config.yml", {Volumes:[{Name:root Path:/hostroot} {Name:data Path:/data}] Services:{HTTP:[{Name:first URL:https://example1.com} {Name:second URL:https://example2.com}] Certificate:[{Name:prim_cert URL:https://example1.com} {Name:second_cert URL:https://example2.com}] File:[{Name:first Path:/tmp/example1.txt} {Name:second Path:/tmp/example2.txt}] Mongo:[{Name:dev URL:mongodb://example.com:27017 OplogMaxDelta:30m0s}] Nginx:[{Name:nginx StatusURL:http://example.com:80}] Program:[{Name:first Path:/usr/bin/example1 Args:[arg1 arg2]} {Name:second Path:/usr/bin/example2 Args:[]}] Docker:[{Name:docker1 URL:unix:///var/run/docker.sock Containers:[reproxy mattermost postgres]} {Name:docker2 URL:tcp://192.168.1.1:4080 Containers:[]}]} fileName:testdata/config.yml}`
+	assert.Equal(t, exp, p.String())
+}
+
+func TestParameters_MarshalServices(t *testing.T) {
+	{
+		p, err := New("testdata/config.yml")
+		require.NoError(t, err)
+		exp := []string{
+			"first:https://example1.com", "second:https://example2.com",
+			"prim_cert:cert://example1.com", "second_cert:cert://example2.com",
+			"docker1:docker:///var/run/docker.sock?containers=reproxy,mattermost,postgres", "docker2:docker://192.168.1.1:4080",
+			"first:file:///tmp/example1.txt", "second:file:///tmp/example2.txt",
+			"dev:mongodb://example.com:27017?oplogMaxDelta=30m0s",
+			"nginx:nginx:http://example.com:80",
+			"first:program:///usr/bin/example1?args=\"arg1 arg2\"", "second:program:///usr/bin/example2",
+		}
+		assert.Equal(t, exp, p.MarshalServices())
+	}
+
+	{ // test mongo with query params
+		p, err := New("testdata/config.yml")
+		require.NoError(t, err)
+		p.Services.Mongo[0].URL = "mongodb://example.com:27017/admin?foo=bar&blah=blah"
+		exp := "dev:mongodb://example.com:27017/admin?foo=bar&blah=blah&oplogMaxDelta=30m0s"
+		res := p.MarshalServices()
+		found := false
+		for _, r := range res {
+			if r == exp {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "expected %s in %v", exp, res)
+	}
+}

app/config/testdata/config.yml

@@ -0,0 +1,24 @@
+volumes:
+  - {name: root, path: /hostroot}
+  - {name: data, path: /data}
+
+services:
+  mongo:
+    - {name: dev, url: mongodb://example.com:27017, oplog_max_delta: 30m}
+  certificate:
+    - {name: prim_cert, url: https://example1.com}
+    - {name: second_cert, url: https://example2.com}
+  docker:
+    - {name: docker1, url: unix:///var/run/docker.sock, containers: [reproxy, mattermost, postgres]}
+    - {name: docker2, url: tcp://192.168.1.1:4080}
+  file:
+    - {name: first, path: /tmp/example1.txt}
+    - {name: second, path: /tmp/example2.txt}
+  http:
+    - {name: first, url: https://example1.com}
+    - {name: second, url: https://example2.com}
+  program:
+    - {name: first, path: /usr/bin/example1, args: [arg1, arg2]}
+    - {name: second, path: /usr/bin/example2}
+  nginx:
+    - {name: nginx, status_url: http://example.com:80}