-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improve prospector state handling (#2840)
Previously each prospector was holding old states initially loaded by the registry file. This was changed to that each prospector only loads the states with a path that matches the glob pattern. This reduces the number of states handled by each prospector and reduces overlap. One consequence of this is that in case a log file "moves" from one prospector to an other through renaming during the runtime of filebeat, no state will be found. But this behaviour is not expected and would lead to other issues already now. The expectation is that a file stays inside the prospector over its full lifetime. In case of a filebeat restart including a config change, it can happen that some states are not managed anymore by a prospector. These "obsolete" states will stay in the registrar until a further config change when a glob pattern would again include these states. As an example: Filebeat is started with the following config. ``` filebeat.prospectors: - paths: ['*.log'] clean_removed: true ``` There is a log file `a.log` and `b.log`. Both files are harvested and states for `a.log` and `b.log` are persisted. Then filebeat is stopped and the config file is modified to. ``` filebeat.prospectors: - paths: ['a.log'] clean_removed: true ``` Filebeat is started again. The prospector will now only handle the state for `a.log`. In case `b.log` is removed from disk, the state for `b.log` will stay in the registry file. In case the config file was with `clean_inactive: 5min`, all TTL are reset on restart to `-2` by the registry and `-1` the prospector or the new `clean_inactive` value. Using `-2` in the registrar can be useful in the future to detect which states are not managed anymore by any prospector. As all TTL are reset on restart, persisting of the TTL is not required anymore but can become useful for cleanup so the information is kept in the registry file. Further changes: * Add tests for matchFile method * Add tests for prospector state init filtering * states are passed to `Prospector.Init` on startup instead of setting it directly in the object * `Prospector.Init` and `Prospectorer.Init` now return an error and filebeat exits on startup problems * Remove lastClean as not needed anymore * Have one log file for each bom file test * Update to new vagrant box with Golang 1.7.3
- Loading branch information
Showing
12 changed files
with
471 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| // +build !windows | ||
|
|
||
| package prospector | ||
|
|
||
| import ( | ||
| "regexp" | ||
| "testing" | ||
|
|
||
| "github.com/elastic/beats/filebeat/input" | ||
| "github.com/elastic/beats/filebeat/input/file" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| var matchTests = []struct { | ||
| file string | ||
| paths []string | ||
| excludeFiles []*regexp.Regexp | ||
| result bool | ||
| }{ | ||
| { | ||
| "test/test.log", | ||
| []string{"test/*"}, | ||
| nil, | ||
| true, | ||
| }, | ||
| { | ||
| "notest/test.log", | ||
| []string{"test/*"}, | ||
| nil, | ||
| false, | ||
| }, | ||
| { | ||
| "test/test.log", | ||
| []string{"test/*.log"}, | ||
| nil, | ||
| true, | ||
| }, | ||
| { | ||
| "test/test.log", | ||
| []string{"test/*.nolog"}, | ||
| nil, | ||
| false, | ||
| }, | ||
| { | ||
| "test/test.log", | ||
| []string{"test/*"}, | ||
| []*regexp.Regexp{regexp.MustCompile("test.log")}, | ||
| false, | ||
| }, | ||
| { | ||
| "test/test.log", | ||
| []string{"test/*"}, | ||
| []*regexp.Regexp{regexp.MustCompile("test2.log")}, | ||
| true, | ||
| }, | ||
| } | ||
|
|
||
| func TestMatchFile(t *testing.T) { | ||
|
|
||
| for _, test := range matchTests { | ||
|
|
||
| p := ProspectorLog{ | ||
| config: prospectorConfig{ | ||
| Paths: test.paths, | ||
| ExcludeFiles: test.excludeFiles, | ||
| }, | ||
| } | ||
|
|
||
| assert.Equal(t, test.result, p.matchesFile(test.file)) | ||
| } | ||
| } | ||
|
|
||
| var initStateTests = []struct { | ||
| states []file.State // list of states | ||
| paths []string // prospector glob | ||
| count int // expected states in prospector | ||
| }{ | ||
| { | ||
| []file.State{ | ||
| file.State{Source: "test"}, | ||
| }, | ||
| []string{"test"}, | ||
| 1, | ||
| }, | ||
| { | ||
| []file.State{ | ||
| file.State{Source: "notest"}, | ||
| }, | ||
| []string{"test"}, | ||
| 0, | ||
| }, | ||
| { | ||
| []file.State{ | ||
| file.State{Source: "test1.log", FileStateOS: file.StateOS{Inode: 1}}, | ||
| file.State{Source: "test2.log", FileStateOS: file.StateOS{Inode: 2}}, | ||
| }, | ||
| []string{"*.log"}, | ||
| 2, | ||
| }, | ||
| { | ||
| []file.State{ | ||
| file.State{Source: "test1.log", FileStateOS: file.StateOS{Inode: 1}}, | ||
| file.State{Source: "test2.log", FileStateOS: file.StateOS{Inode: 2}}, | ||
| }, | ||
| []string{"test1.log"}, | ||
| 1, | ||
| }, | ||
| { | ||
| []file.State{ | ||
| file.State{Source: "test1.log", FileStateOS: file.StateOS{Inode: 1}}, | ||
| file.State{Source: "test2.log", FileStateOS: file.StateOS{Inode: 2}}, | ||
| }, | ||
| []string{"test.log"}, | ||
| 0, | ||
| }, | ||
| { | ||
| []file.State{ | ||
| file.State{Source: "test1.log", FileStateOS: file.StateOS{Inode: 1}}, | ||
| file.State{Source: "test2.log", FileStateOS: file.StateOS{Inode: 1}}, | ||
| }, | ||
| []string{"*.log"}, | ||
| 1, // Expecting only 1 state because of some inode (this is only a theoretical case) | ||
| }, | ||
| } | ||
|
|
||
| // TestInit checks that the correct states are in a prospector after the init phase | ||
| // This means only the ones that match the glob and not exclude files | ||
| func TestInit(t *testing.T) { | ||
|
|
||
| for _, test := range initStateTests { | ||
| p := ProspectorLog{ | ||
| Prospector: &Prospector{ | ||
| states: &file.States{}, | ||
| outlet: TestOutlet{}, | ||
| }, | ||
| config: prospectorConfig{ | ||
| Paths: test.paths, | ||
| }, | ||
| } | ||
| err := p.Init(test.states) | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, test.count, p.Prospector.states.Count()) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| // TestOutlet is an empty outlet for testing | ||
| type TestOutlet struct{} | ||
|
|
||
| func (o TestOutlet) OnEvent(event *input.Event) bool { return true } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| // +build windows | ||
|
|
||
| package prospector | ||
|
|
||
| import ( | ||
| "regexp" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| var matchTestsWindows = []struct { | ||
| file string | ||
| paths []string | ||
| excludeFiles []*regexp.Regexp | ||
| result bool | ||
| }{ | ||
| { | ||
| "C:\\\\hello\\test\\test.log", // Path are always in windows format | ||
| []string{"C:\\\\hello/test/*.log"}, // Globs can also be with forward slashes | ||
| nil, | ||
| true, | ||
| }, | ||
| { | ||
| "C:\\\\hello\\test\\test.log", // Path are always in windows format | ||
| []string{"C:\\\\hello\\test/*.log"}, // Globs can also be mixed | ||
| nil, | ||
| true, | ||
| }, | ||
| } | ||
|
|
||
| // TestMatchFileWindows test if match works correctly on windows | ||
| // Separate test are needed on windows because of automated path conversion | ||
| func TestMatchFileWindows(t *testing.T) { | ||
|
|
||
| for _, test := range matchTestsWindows { | ||
|
|
||
| p := ProspectorLog{ | ||
| config: prospectorConfig{ | ||
| Paths: test.paths, | ||
| ExcludeFiles: test.excludeFiles, | ||
| }, | ||
| } | ||
|
|
||
| assert.Equal(t, test.result, p.matchesFile(test.file)) | ||
| } | ||
| } |
Oops, something went wrong.