change from using "container" to "app"

* in logdna container is only available in some user cases, _app is always available

CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changes
+- `containers` in the config file is renamed to `apps` in order to be more flexible
 
 ## [0.2.0] - 2018-02-15
 ### Added

README.md

@@ -31,15 +31,14 @@ All of these settings are currently required.
 LogDirectory = 'logs' # temp local directory to download logs to and store results, this directory will be created if it doesn't exist
 ```
 
-### Containers
-Note: the term container here is used to be consistent with LogDNA's terminology
+### Apps
 ```
-[[containers]]
-    Name = "production-app"  # name of container
+[[apps]]
+    Name = "production-app"  # name of app
     Regex = '^([\d.]+) - \[([^\]]*)\] - - \[([^\]]*)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)" (\d+) ([\d.]+)  ([\d.:]+) (\d+) ([\d.]+) (\d+)$'  # regex used to pull parts of logs out, currently we don't use any named capture groups
     TimeGroup = 3 # the number of the capture group (1-based) in above regex that holds the time component of the log
     TimeFormat = "2/Jan/2006:15:04:05 -0700" # the format of the time field using https://golang.org/pkg/time/#Parse
-    [[containers.excludes]]  # an array of tables for exclusions
+    [[apps.excludes]]  # an array of tables for exclusions
         Group = 4  # the number of the capture group in above regex to be used in the exclusion check
         Contains = "ping" # excludes logs if the value here is contained in the string in the above capture group
 ```
@@ -58,7 +57,7 @@ LogPrefix = "a7112abc9d" # each archive file starts with a prefix specific to yo
 
 #### TemplateTable
 
-Currently the schema for the BigQuery table is specified by creating a table with the schema that matches the regex. This is likely to change in the future as we may need a different schema for each container above.
+Currently the schema for the BigQuery table is specified by creating a table with the schema that matches the regex. This is likely to change in the future as we may need a different schema for each app above.
 
 ## Usage
 

config.go

@@ -2,11 +2,12 @@ package main
 
 import (
 	"fmt"
-	"github.com/pelletier/go-toml"
-	"github.com/pkg/errors"
 	"io/ioutil"
 	"os"
 	"regexp"
+
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
 )
 
 // Storage holds the configuration for [storage] section of the toml config.
@@ -25,16 +26,16 @@ type GCP struct {
 	LogBucket       string
 }
 
-// Exclude holds the configuration for the [[containers.excludes]] subsection
+// Exclude holds the configuration for the [[apps.excludes]] subsection
 // of the toml config.
 type Exclude struct {
 	Group    int
 	Contains string
 }
 
-// Container holds the configuration for a single entry in the [[containers]]
+// App holds the configuration for a single entry in the [[apps]]
 // section of the toml config.
-type Container struct {
+type App struct {
 	Name          string
 	Regex         string
 	CompiledRegex *regexp.Regexp
@@ -45,32 +46,32 @@ type Container struct {
 
 // Configuration holds the full configuration loaded from the toml config file.
 type Configuration struct {
-	Storage    Storage
-	GCP        GCP
-	Containers []Container
+	Storage Storage
+	GCP     GCP
+	Apps    []App
 }
 
-func (cfg *Configuration) getContainer(c string) (Container, error) {
-	for _, container := range cfg.Containers {
-		if c == container.Name {
-			return container, nil
+func (cfg *Configuration) getApp(c string) (App, error) {
+	for _, app := range cfg.Apps {
+		if c == app.Name {
+			return app, nil
 		}
 	}
-	return Container{}, errors.New("Container not found")
+	return App{}, errors.New("App not found")
 }
 
-func (cfg *Configuration) extractContainerNames() (set map[string]struct{}) {
-	set = make(map[string]struct{}, len(cfg.Containers))
-	for _, container := range cfg.Containers {
-		set[container.Name] = struct{}{}
+func (cfg *Configuration) extractAppNames() (set map[string]struct{}) {
+	set = make(map[string]struct{}, len(cfg.Apps))
+	for _, app := range cfg.Apps {
+		set[app.Name] = struct{}{}
 	}
 	return set
 }
 
 func (cfg *Configuration) compileRegex() {
-	for i, c := range cfg.Containers {
+	for i, c := range cfg.Apps {
 		cmp := regexp.MustCompile(c.Regex)
-		cfg.Containers[i].CompiledRegex = cmp
+		cfg.Apps[i].CompiledRegex = cmp
 	}
 }
 

config_test.go

@@ -34,37 +34,36 @@ func TestReadConfig(t *testing.T) {
 	os.Remove(f)
 }
 
-func TestGetContainers(t *testing.T) {
-	// build a config with 1 Containers
-	c1 := Container{Name: "Container 1"}
+func TestGetApp(t *testing.T) {
+	// build a config with 1 App
+	a1 := App{Name: "App 1"}
 	cfg := &Configuration{
-		Containers: []Container{c1},
+		Apps: []App{a1},
 	}
-	_, err := cfg.getContainer("Container That Doesn't Exist")
+	_, err := cfg.getApp("App That Doesn't Exist")
 	if err == nil {
-		t.Errorf("Err not returned when container doesn't exist")
+		t.Errorf("Err not returned when app doesn't exist")
 	}
 
-	c1Retrieved, _ := cfg.getContainer("Container 1")
-	if !cmp.Equal(c1, c1Retrieved) {
-		t.Errorf("c1 was not equal")
+	a1Retrieved, _ := cfg.getApp("App 1")
+	if !cmp.Equal(a1, a1Retrieved) {
+		t.Errorf("a1 was not equal")
 	}
 }
 
-
 func TestCompileRegex(t *testing.T) {
-	c1 := Container{
-		Name:  "Container 1",
+	a1 := App{
+		Name:  "App 1",
 		Regex: `([\d.]+) - \[([^\]]*)\] - - \[([^\]]*)\]`,
 	}
 	cfg := &Configuration{
-		Containers: []Container{c1},
+		Apps: []App{a1},
 	}
-	if cfg.Containers[0].CompiledRegex != nil {
+	if cfg.Apps[0].CompiledRegex != nil {
 		t.Errorf("compile regex is not nil but compile hasn't been called yet")
 	}
 	cfg.compileRegex()
-	if cfg.Containers[0].CompiledRegex == nil {
+	if cfg.Apps[0].CompiledRegex == nil {
 		t.Errorf("compile regex is nil")
 	}
 }
@@ -97,16 +96,15 @@ func TestSetupDir(t *testing.T) {
 	}
 }
 
-func TestExtractContainerNames(t *testing.T) {
+func TestExtractAppNames(t *testing.T) {
 	cfg := &Configuration{
-		Containers: []Container{{Name: "c1"}, {Name: "c2"}},
+		Apps: []App{{Name: "a1"}, {Name: "a2"}},
 	}
-	names := cfg.extractContainerNames()
-	expected := make(map[string]struct{}, len(cfg.Containers))
-	expected["c1"] = struct{}{}
-	expected["c2"] = struct{}{}
+	names := cfg.extractAppNames()
+	expected := make(map[string]struct{}, len(cfg.Apps))
+	expected["a1"] = struct{}{}
+	expected["a2"] = struct{}{}
 	if !cmp.Equal(names, expected) {
 		t.Errorf("expected names to equal %v, received %v instead", expected, names)
 	}
 }
-

example.toml

@@ -1,12 +1,12 @@
 [storage]
 LogDirectory = 'logs'
 
-[[containers]]
+[[apps]]
     Name = "production-app"
     Regex = '^([\d.]+) - \[([^\]]*)\] - - \[([^\]]*)\] "([^"]*)" (\d+) (\d+) "([^"]*)" "([^"]*)" (\d+) ([\d.]+) ([\d.:]+) (\d+) ([\d.]+) (\d+)$'
     TimeGroup = 3
     TimeFormat = "2/Jan/2006:15:04:05 -0700"
-    [[containers.excludes]]
+    [[apps.excludes]]
         Group = 4
         Contains = "ping"
 

main.go

@@ -21,21 +21,21 @@ import (
 	"google.golang.org/api/option"
 )
 
-var version = "0.2.0-dev"
+var version = "0.3.0-dev"
 
 type DNAQuery struct {
 	*Configuration
-	containerNames map[string]struct{}
+	appNames map[string]struct{}
 }
 
 func NewDNAQuery(cfg *Configuration) (*DNAQuery, error) {
-	if len(cfg.Containers) < 1 {
-		return nil, errors.New("Configuration needs at least 1 container")
+	if len(cfg.Apps) < 1 {
+		return nil, errors.New("Configuration needs at least 1 app")
 	}
 
 	dna := &DNAQuery{
-		Configuration:  cfg,
-		containerNames: cfg.extractContainerNames(),
+		Configuration: cfg,
+		appNames:      cfg.extractAppNames(),
 	}
 	dna.compileRegex()
 	err := dna.setupDirectory()
@@ -77,10 +77,10 @@ func (d *DNAQuery) readLine(path string) chan [2]string {
 		for scanner.Scan() {
 			lineCount++
 			data := scanner.Bytes()
-			container, _ := jsonparser.GetString(data, "container")
-			if _, ok := d.containerNames[container]; ok {
+			app, _ := jsonparser.GetString(data, "_app")
+			if _, ok := d.appNames[app]; ok {
 				line, _ := jsonparser.GetString(data, "_line")
-				ch <- [2]string{container, line}
+				ch <- [2]string{app, line}
 			}
 		}
 		if err := scanner.Err(); err != nil {
@@ -102,13 +102,13 @@ func (d *DNAQuery) processLine(path string, ch chan [2]string) {
 	nMatches := 0
 	nSkipped := 0
 	for r := range ch {
-		container := r[0]
+		app := r[0]
 		line := r[1]
 		var record []string
-		record = append(record, container)
-		c, err := d.getContainer(container)
+		record = append(record, app)
+		c, err := d.getApp(app)
 		if err != nil {
-			log.Println("Can't find container config in processLine:", container)
+			log.Println("Can't find app config in processLine:", app)
 			continue
 		}
 		result := c.CompiledRegex.FindStringSubmatch(line)

main_test.go

@@ -11,11 +11,11 @@ import (
 func TestNewDNAQuery(t *testing.T) {
 	_, err := NewDNAQuery(&Configuration{})
 	if err == nil {
-		t.Errorf("should error if 0 Containers in config")
+		t.Errorf("should error if 0 Apps in config")
 	}
-	c1 := Container{Name: "Container 1"}
+	c1 := App{Name: "App 1"}
 	cfg := &Configuration{
-		Containers: []Container{c1},
+		Apps: []App{c1},
 	}
 	_, err = NewDNAQuery(cfg)
 	if err == nil {
@@ -56,31 +56,31 @@ func TestCleanupFiles(t *testing.T) {
 }
 
 func TestProcessLine(t *testing.T) {
-	c1 := Container{
-		Name:       "container1",
+	c1 := App{
+		Name:       "app1",
 		Regex:      `^([\d.]+) \[([^\]]*)\] - "([^"]*)" (\d+)`,
 		TimeGroup:  2,
 		TimeFormat: "2/Jan/2006:15:04:05 -0700",
 	}
-	c2 := Container{
-		Name:       "container2",
+	c2 := App{
+		Name:       "app2",
 		Regex:      `^([\d.]+) - \[([^\]]*)\] - "([^"]*)" (\d+)`,
 		TimeGroup:  2,
 		TimeFormat: "2006-01-02T15:04:05-0700",
 		Excludes:   []Exclude{{Group: 3, Contains: "ping"}},
 	}
 	// invalid configuration, should have code to detect this at start up, for now
 	// make sure code handles it
-	c3 := Container{
-		Name:       "container3",
+	c3 := App{
+		Name:       "app3",
 		Regex:      `^([\d.]+)`,
 		TimeGroup:  2,
 		TimeFormat: "2006-01-02T15:04:05-0700",
 		Excludes:   []Exclude{{Group: 3, Contains: "ping"}},
 	}
 	cfg := &Configuration{
-		Containers: []Container{c1, c2, c3},
-		Storage:    Storage{LogDirectory: "/tmp/"},
+		Apps:    []App{c1, c2, c3},
+		Storage: Storage{LogDirectory: "/tmp/"},
 	}
 	dna, err := NewDNAQuery(cfg)
 	if err != nil {
@@ -90,16 +90,16 @@ func TestProcessLine(t *testing.T) {
 	outfile := "output.csv"
 	go dna.processLine(outfile, ch)
 	// regular, expect normal operation
-	ch <- [2]string{"container1", `123.123.123.123 [13/Nov/2017:13:23:01 -0000] - "GET view.json" 200`}
-	ch <- [2]string{"container1", `123.123.123.123 [13/Nov/2017:13:23:04 -0000] - "GET ping.json" 200`}
-	ch <- [2]string{"container2", `2.1.5.3 - [2017-12-03T13:23:04-0500] - "GET ping.json" 200`}
-	ch <- [2]string{"container2", `2.1.5.3 - [2017-12-03T13:23:04-0500] - "GET view.json" 200`}
+	ch <- [2]string{"app1", `123.123.123.123 [13/Nov/2017:13:23:01 -0000] - "GET view.json" 200`}
+	ch <- [2]string{"app1", `123.123.123.123 [13/Nov/2017:13:23:04 -0000] - "GET ping.json" 200`}
+	ch <- [2]string{"app2", `2.1.5.3 - [2017-12-03T13:23:04-0500] - "GET ping.json" 200`}
+	ch <- [2]string{"app2", `2.1.5.3 - [2017-12-03T13:23:04-0500] - "GET view.json" 200`}
 	// case where exclusion and time groups are more then regex
-	ch <- [2]string{"container3", `2.1.5.3`}
-	// case where there is no container registered
-	ch <- [2]string{"container", `2.1.5.3 - [2017-12-03T13:23:04-0500] - "GET view.json" 200`}
+	ch <- [2]string{"app3", `2.1.5.3`}
+	// case where there is no app registered
+	ch <- [2]string{"app", `2.1.5.3 - [2017-12-03T13:23:04-0500] - "GET view.json" 200`}
 	// case where the log line doesn't match regex
-	ch <- [2]string{"container1", `error`}
+	ch <- [2]string{"app1", `error`}
 	close(ch)
 
 	// sleep a bit for goroutine to finish up once channel is closed