From f56d995af981b737b6b903ed0b2b4218a64ac76e Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 30 May 2016 11:27:12 +0200 Subject: [PATCH 1/5] Stop following symlinks. Previously symlinks were followed. This had the consequence if a symlink and the file itself existed, the file was read twice. Now symlinks are not followed anymore. This closes #1686 --- CHANGELOG.asciidoc | 4 ++- filebeat/crawler/prospector_log.go | 6 +++- filebeat/tests/system/test_prospector.py | 38 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a36c8a50e61..8e48d076854 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -22,6 +22,8 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d *Filebeat* +- Stop following symlink. Symlinks are now ignored: {pull}1686[1686] + *Winlogbeat* @@ -79,7 +81,7 @@ https://github.com/elastic/beats/compare/v5.0.0alpha2...v5.0.0-alpha3[View commi *Affecting all Beats* - All configuration settings under `shipper:` are moved to be top level configuration settings. I.e. - `shipper.name:` becomes `name:` in the configuration file. #1570 + `shipper.name:` becomes `name:` in the configuration file. {pull}1570[1570] *Packetbeat* diff --git a/filebeat/crawler/prospector_log.go b/filebeat/crawler/prospector_log.go index 281aa5c8b7b..13f03cbf67d 100644 --- a/filebeat/crawler/prospector_log.go +++ b/filebeat/crawler/prospector_log.go @@ -79,11 +79,15 @@ func (p *ProspectorLog) getFiles() map[string]os.FileInfo { } // Stat the file, following any symlinks. - fileinfo, err := os.Stat(file) + fileinfo, err := os.Lstat(file) if err != nil { logp.Debug("prospector", "stat(%s) failed: %s", file, err) continue } + if fileinfo.Mode()&os.ModeSymlink != 0 { + logp.Debug("prospector", "File %s skipped as it is a symlink.", file) + continue + } if fileinfo.IsDir() { logp.Debug("prospector", "Skipping directory: %s", file) diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index 18d3e7c88ed..b8ef4b658ff 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -514,3 +514,41 @@ def test_close_older_file_rotation_and_removal(self): max_timeout=10) filebeat.check_kill_and_wait() + + def test_skip_symlinks(self): + """ + Test that symlinks are skipped + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + ) + + os.mkdir(self.working_dir + "/log/") + testfile = self.working_dir + "/log/test-2016.log" + symlink_file = self.working_dir + "/log/test.log" + + # write first line + with open(testfile, 'a') as file: + file.write("Hello world\n") + + os.symlink(testfile, symlink_file) + + filebeat = self.start_beat() + + # wait for file to be skipped + self.wait_until( + lambda: self.log_contains("skipped as it is a symlink"), + max_timeout=10) + + # wait for log to be read + self.wait_until( + lambda: self.output_has(lines=1), + max_timeout=15) + + time.sleep(5) + filebeat.check_kill_and_wait() + + data = self.read_output() + + # Make sure there is only one entry, means it didn't follow the symlink + assert len(data) == 1 From 0196558d5235cbfdc5469cc8a2067a4d5ed488a9 Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 30 May 2016 12:44:00 +0200 Subject: [PATCH 2/5] Add symlink for windows --- filebeat/tests/system/test_prospector.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index b8ef4b658ff..441f5f617a8 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -531,7 +531,11 @@ def test_skip_symlinks(self): with open(testfile, 'a') as file: file.write("Hello world\n") - os.symlink(testfile, symlink_file) + if os.name == "nt": + import win32file + win32file.CreateSymbolicLink(testfile, symlink_file, 1) + else: + os.symlink(testfile, symlink_file) filebeat = self.start_beat() From ae803d9363f30fcdfdc3c985bb59097822f7f843 Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 30 May 2016 13:17:45 +0200 Subject: [PATCH 3/5] Turn around params --- filebeat/tests/system/test_prospector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index 441f5f617a8..aee1920689b 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -533,7 +533,7 @@ def test_skip_symlinks(self): if os.name == "nt": import win32file - win32file.CreateSymbolicLink(testfile, symlink_file, 1) + win32file.CreateSymbolicLink(symlink_file, testfile, 1) else: os.symlink(testfile, symlink_file) From 795487434bc23c627214217234c361090f031117 Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 31 May 2016 14:15:41 +0200 Subject: [PATCH 4/5] Remove symlink comment --- filebeat/crawler/prospector_log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/crawler/prospector_log.go b/filebeat/crawler/prospector_log.go index 13f03cbf67d..c0faa75d74a 100644 --- a/filebeat/crawler/prospector_log.go +++ b/filebeat/crawler/prospector_log.go @@ -78,12 +78,12 @@ func (p *ProspectorLog) getFiles() map[string]os.FileInfo { continue } - // Stat the file, following any symlinks. fileinfo, err := os.Lstat(file) if err != nil { logp.Debug("prospector", "stat(%s) failed: %s", file, err) continue } + // Check if file is symlink if fileinfo.Mode()&os.ModeSymlink != 0 { logp.Debug("prospector", "File %s skipped as it is a symlink.", file) continue From d9dbadf8d47c9b1b0048d1fbd672af98794927ab Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 31 May 2016 14:42:22 +0200 Subject: [PATCH 5/5] Fix for windows symlink --- filebeat/tests/system/test_prospector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index aee1920689b..da9c9c0d977 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -533,7 +533,7 @@ def test_skip_symlinks(self): if os.name == "nt": import win32file - win32file.CreateSymbolicLink(symlink_file, testfile, 1) + win32file.CreateSymbolicLink(symlink_file, testfile, 0) else: os.symlink(testfile, symlink_file)