From 15192adc0c6a21c74f4e889f7e6264e4e2bfff44 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 20 Apr 2016 23:34:08 +0200 Subject: [PATCH 1/3] Added path.logs and made logging to files the default This adds the `-path.logs` CLI flag and corresponding config file setting. By default it's set to `{path.home}/logs`. The rotating file writer now writes to this path by default. For DEB/RPM, the logs path is set to `/var/log/{beatname}` which matches the previous default location. However, this is a breaking change because the previous default was to send the logs to syslog. Part of #1371. --- dev-tools/packer/platforms/centos/init.j2 | 2 +- dev-tools/packer/platforms/centos/systemd.j2 | 2 +- dev-tools/packer/platforms/debian/init.j2 | 2 +- dev-tools/packer/platforms/debian/systemd.j2 | 2 +- libbeat/logp/logp.go | 21 ++---- libbeat/paths/paths.go | 22 ++++-- libbeat/paths/paths_test.go | 72 ++++++++++++++++++++ 7 files changed, 99 insertions(+), 24 deletions(-) diff --git a/dev-tools/packer/platforms/centos/init.j2 b/dev-tools/packer/platforms/centos/init.j2 index 823ac3b9752..f9e5c9214bc 100755 --- a/dev-tools/packer/platforms/centos/init.j2 +++ b/dev-tools/packer/platforms/centos/init.j2 @@ -24,7 +24,7 @@ export PATH [ -f /etc/sysconfig/{{.beat_name}} ] && . /etc/sysconfig/{{.beat_name}} pidfile=${PIDFILE-/var/run/{{.beat_name}}.pid} agent=${BEATS_AGENT-/usr/share/{{.beat_name}}/bin/{{.beat_name}}} -args="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}}" +args="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}}" test_args="-e -configtest" wrapper="/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god" wrapperopts="-r / -n -p $pidfile" diff --git a/dev-tools/packer/platforms/centos/systemd.j2 b/dev-tools/packer/platforms/centos/systemd.j2 index 8ca1784bac6..457f289fa83 100644 --- a/dev-tools/packer/platforms/centos/systemd.j2 +++ b/dev-tools/packer/platforms/centos/systemd.j2 @@ -5,7 +5,7 @@ Wants=network-online.target After=network-online.target [Service] -ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} +ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}} Restart=always [Install] diff --git a/dev-tools/packer/platforms/debian/init.j2 b/dev-tools/packer/platforms/debian/init.j2 index 6a2816924b5..f339f5bf6a7 100755 --- a/dev-tools/packer/platforms/debian/init.j2 +++ b/dev-tools/packer/platforms/debian/init.j2 @@ -17,7 +17,7 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="{{.beat_description}}" NAME="{{.beat_name}}" DAEMON=/usr/share/{{.beat_name}}/bin/$NAME -DAEMON_ARGS="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}}" +DAEMON_ARGS="-c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}}" TEST_ARGS="-e -configtest" PIDFILE=/var/run/{{.beat_name}}.pid WRAPPER="/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god" diff --git a/dev-tools/packer/platforms/debian/systemd.j2 b/dev-tools/packer/platforms/debian/systemd.j2 index 8ca1784bac6..457f289fa83 100644 --- a/dev-tools/packer/platforms/debian/systemd.j2 +++ b/dev-tools/packer/platforms/debian/systemd.j2 @@ -5,7 +5,7 @@ Wants=network-online.target After=network-online.target [Service] -ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} +ExecStart=/usr/share/{{.beat_name}}/bin/{{.beat_name}} -c /etc/{{.beat_name}}/{{.beat_name}}.yml -path.home /usr/share/{{.beat_name}} -path.config /etc/{{.beat_name}} -path.data /var/lib/{{.beat_name}} -path.logs /var/log/{{.beat_name}} Restart=always [Install] diff --git a/libbeat/logp/logp.go b/libbeat/logp/logp.go index b29cb17800e..762abc894ba 100644 --- a/libbeat/logp/logp.go +++ b/libbeat/logp/logp.go @@ -5,8 +5,9 @@ import ( "fmt" "io/ioutil" "log" - "runtime" "strings" + + "github.com/elastic/beats/libbeat/paths" ) // cmd line flags @@ -57,29 +58,19 @@ func Init(name string, config *Logging) error { logLevel = LOG_DEBUG } - var defaultToFiles, defaultToSyslog bool - var defaultFilePath string - if runtime.GOOS == "windows" { - // always disabled on windows - defaultToSyslog = false - defaultToFiles = true - defaultFilePath = fmt.Sprintf("C:\\ProgramData\\%s\\Logs", name) - } else { - defaultToSyslog = true - defaultToFiles = false - defaultFilePath = fmt.Sprintf("/var/log/%s", name) - } + // default log location is in the logs path + defaultFilePath := paths.Resolve(paths.Logs, "") var toSyslog, toFiles bool if config.ToSyslog != nil { toSyslog = *config.ToSyslog } else { - toSyslog = defaultToSyslog + toSyslog = false } if config.ToFiles != nil { toFiles = *config.ToFiles } else { - toFiles = defaultToFiles + toFiles = true } // toStderr disables logging to syslog/files diff --git a/libbeat/paths/paths.go b/libbeat/paths/paths.go index d910726c8ed..8575824d597 100644 --- a/libbeat/paths/paths.go +++ b/libbeat/paths/paths.go @@ -25,20 +25,20 @@ import ( "fmt" "os" "path/filepath" - - "github.com/elastic/beats/libbeat/logp" ) var ( homePath = flag.String("path.home", "", "Home path") configPath = flag.String("path.config", "", "Configuration path") dataPath = flag.String("path.data", "", "Data path") + logsPath = flag.String("path.logs", "", "Logs path") ) type Path struct { Home string Config string Data string + Logs string } // FileType is an enumeration type representing the file types. @@ -49,6 +49,7 @@ const ( Home FileType = "home" Config FileType = "config" Data FileType = "data" + Logs FileType = "logs" ) // Paths is the Path singleton on which the top level functions from this @@ -91,6 +92,7 @@ func (paths *Path) initPaths(cfg *Path) error { paths.Home = cfg.Home paths.Config = cfg.Config paths.Data = cfg.Data + paths.Logs = cfg.Logs // overwrite paths from CLI flags if homePath != nil && len(*homePath) > 0 { @@ -102,6 +104,9 @@ func (paths *Path) initPaths(cfg *Path) error { if dataPath != nil && len(*dataPath) > 0 { paths.Data = *dataPath } + if logsPath != nil && len(*logsPath) > 0 { + paths.Logs = *logsPath + } // default for the home path is the binary location if len(paths.Home) == 0 { @@ -123,6 +128,11 @@ func (paths *Path) initPaths(cfg *Path) error { paths.Data = filepath.Join(paths.Home, "data") } + // default for logs path + if len(paths.Logs) == 0 { + paths.Logs = filepath.Join(paths.Home, "logs") + } + return nil } @@ -142,8 +152,10 @@ func (paths *Path) Resolve(fileType FileType, path string) string { return filepath.Join(paths.Config, path) case Data: return filepath.Join(paths.Data, path) + case Logs: + return filepath.Join(paths.Logs, path) default: - logp.WTF("Unknown file type: %s", fileType) + panic(fmt.Sprintf("Unknown file type: %s", fileType)) return "" } } @@ -157,6 +169,6 @@ func Resolve(fileType FileType, path string) string { // String returns a textual representation func (paths *Path) String() string { - return fmt.Sprintf("Home path: [%s] Config path: [%s] Data path: [%s]", - paths.Home, paths.Config, paths.Data) + return fmt.Sprintf("Home path: [%s] Config path: [%s] Data path: [%s] Logs path: [%s]", + paths.Home, paths.Config, paths.Data, paths.Logs) } diff --git a/libbeat/paths/paths_test.go b/libbeat/paths/paths_test.go index 9128a2326af..e8f30d98d29 100644 --- a/libbeat/paths/paths_test.go +++ b/libbeat/paths/paths_test.go @@ -60,6 +60,7 @@ func TestHomePath(t *testing.T) { } for _, test := range tests { + t.Log("Executing test", test) homePath = test.CLIHome cfg := Path{Home: test.CfgHome} assert.NoError(t, Paths.initPaths(&cfg)) @@ -126,6 +127,7 @@ func TestDataPath(t *testing.T) { } for _, test := range tests { + t.Log("Executing test", test) homePath = test.CLIHome dataPath = test.CLIData cfg := Path{Home: test.CfgHome, Data: test.CfgData} @@ -135,3 +137,73 @@ func TestDataPath(t *testing.T) { } } + +func TestLogsPath(t *testing.T) { + type io struct { + CLIHome *string // cli flag home setting + CfgHome string // config file home setting + CLILogs *string // cli flag for data setting + CfgLogs string // config file data setting + Path string // requested path + ResultLogs string // expected logs path + } + + binDir, err := filepath.Abs(filepath.Dir(os.Args[0])) + assert.NoError(t, err) + tmp := "/tmp/" + root := "/root/" + + tests := []io{ + { + CLIHome: nil, + CfgHome: "", + CLILogs: nil, + CfgLogs: "", + Path: "", + ResultLogs: filepath.Join(binDir, "logs"), + }, + { + CLIHome: nil, + CfgHome: "", + CLILogs: nil, + CfgLogs: "", + Path: "test", + ResultLogs: filepath.Join(binDir, "logs", "test"), + }, + { + CLIHome: nil, + CfgHome: "/tmp/", + CLILogs: nil, + CfgLogs: "/root/logs", + Path: "", + ResultLogs: "/root/logs", + }, + { + CLIHome: &tmp, + CfgHome: "", + CLILogs: nil, + CfgLogs: "/root/logs", + Path: "", + ResultLogs: "/root/logs", + }, + { + CLIHome: &tmp, + CfgHome: "", + CLILogs: &root, + CfgLogs: "/root/logs", + Path: "", + ResultLogs: "/root", + }, + } + + for _, test := range tests { + t.Log("Executing test", test) + homePath = test.CLIHome + logsPath = test.CLILogs + cfg := Path{Home: test.CfgHome, Logs: test.CfgLogs} + assert.NoError(t, Paths.initPaths(&cfg)) + + assert.Equal(t, test.ResultLogs, Resolve(Logs, test.Path)) + } + +} From be54fa4bf92a57d24cff4b6d44eeafcdd288d2c0 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 20 Apr 2016 23:47:37 +0200 Subject: [PATCH 2/3] Updated CHANGELOG --- CHANGELOG.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f5bdc228673..77a892496da 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -16,6 +16,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d - Add scripts for managing the dashboards of a single Beat {pull}1359[1359] - On DEB/RPM installations, the binary files are now found under `/usr/share/{{beat_name}}/bin`, not in `/usr/bin`. {pull}1385[1385] +- The logs are written by default to self rotating files, instead of syslog. {pull}1371[1371] *Packetbeat* @@ -57,11 +58,15 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d - Failover and load balancing support in redis output. {pull}1353[1353] - Multiple-worker per host support for redis output. {pull}1353[1353] - Added ability to escape `${x}` in config files to avoid environment variable expansion {pull}1389[1389] +- Configuration options and CLI flags for setting the home, data and config paths. {pull}1373[1373] +- Configuration options and CLI flags for setting the default logs path. {pull}1437[1437] *Packetbeat* + - Add support for OpenBSD. {pull}1347[1347] *Topbeat* + - Add support for OpenBSD. {pull}1403[1403] *Filebeat* @@ -73,7 +78,6 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* -* Configuration options and CLI flags for setting the home, data and config paths. {pull}1373[1373] *Packetbeat* From bf8dda33c2aa8e1d53189f2a153034c0c9fcce66 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 21 Apr 2016 00:33:44 +0200 Subject: [PATCH 3/3] Removed unreachable code --- libbeat/paths/paths.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/paths/paths.go b/libbeat/paths/paths.go index 8575824d597..4019f6b3593 100644 --- a/libbeat/paths/paths.go +++ b/libbeat/paths/paths.go @@ -156,7 +156,6 @@ func (paths *Path) Resolve(fileType FileType, path string) string { return filepath.Join(paths.Logs, path) default: panic(fmt.Sprintf("Unknown file type: %s", fileType)) - return "" } }