From ed256ef15f4121484046301a16b56981b7f48cd8 Mon Sep 17 00:00:00 2001 From: Jesse Zoldak Date: Wed, 30 Jul 2014 09:56:33 -0400 Subject: [PATCH 1/5] Add tests for jenkins agent check --- tests/test_jenkins.py | 136 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 tests/test_jenkins.py diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py new file mode 100644 index 0000000000..358d6d77cd --- /dev/null +++ b/tests/test_jenkins.py @@ -0,0 +1,136 @@ +import unittest +import os +from collections import defaultdict +import datetime +import tempfile +import shutil +import logging + +from nose.tools import raises +from tests.common import get_check + +logger = logging.getLogger(__file__) + +DATETIME_FORMAT = '%Y-%m-%d_%H-%M-%S' +LOG_DATA = 'Finished: SUCCESS' + +BUILD_METADATA = """ + + 20783 + SUCCESS + 487 + +""" +NO_RESULT_YET_METADATA = """ + + 20783 + 487 + +""" + +CONFIG = """ +init_config: + +instances: + - name: default + jenkins_home: +""" + +class TestJenkins(unittest.TestCase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + self.config_yaml = CONFIG.replace('', self.tmp_dir) + + def tearDown(self): + # Clean up the temp directory + shutil.rmtree(self.tmp_dir) + + def _create_builds(self, metadata): + # As coded, the jenkins dd agent needs more than one result + # in order to get the last valid build. + today = datetime.date.today() + yesterday = today - datetime.timedelta(days=1) + + old_date = yesterday.strftime(DATETIME_FORMAT) + todays_date = today.strftime(DATETIME_FORMAT) + + self._create_build(old_date, metadata) + self._create_build(todays_date, metadata) + + def _create_check(self): + # Create the jenkins check + self.check, instances = get_check('jenkins', self.config_yaml) + self.instance = instances[0] + + def _create_build(self, datestring, metadata): + # The jenkins dd agent requires the build metadata file and a log file of results + build_dir = os.path.join(self.tmp_dir, 'jobs', 'foo', 'builds', datestring) + os.makedirs(build_dir) + + metadata_file = open(os.path.join(build_dir, 'build.xml'), 'w+b') + log_file = open(os.path.join(build_dir, 'log'), 'w+b') + + log_data = LOG_DATA + self._write_file(log_file, log_data) + + build_metadata = metadata + self._write_file(metadata_file, build_metadata) + + def _write_file(self, log_file, log_data): + log_file.write(log_data) + log_file.flush() + + def testParseBuildLog(self): + """ + Test doing a jenkins check. This will parse the logs but since there was no + previous high watermark no event will be created. + """ + self._create_builds(BUILD_METADATA) + self._create_check() + self.check.check(self.instance) + + # The check method does not return anything, so this testcase passes + # if the high_watermark was set and no exceptions were raised. + self.assertTrue(self.check.high_watermarks[self.instance['name']]['foo'] > 0) + + def testCheckCreatesEvents(self): + """ + Test that a successful build will create metrics to report in. + """ + self._create_builds(BUILD_METADATA) + self._create_check() + + # Set the high_water mark so that the next check will create events + self.check.high_watermarks['default'] = defaultdict(lambda: 0) + + # Do a check + self.check.check(self.instance) + + results = self.check.get_metrics() + metrics = [r[0] for r in results] + + assert 'jenkins.job.success' in metrics + assert 'jenkins.job.duration' in metrics + assert len(metrics) == 2 + + # This failure is expected due to a bug in the current code. + # TODO: fix the logic in jenkins.py and update this testcase with + # correct expectations. + @raises(KeyError) + def testCheckWithRunningBuild(self): + """ + Test under the conditions of a jenkins build still running. + The build.xml file will exist but it will not yet have a result. + """ + self._create_builds(NO_RESULT_YET_METADATA) + self._create_check() + + # Set the high_water mark so that the next check will create events + self.check.high_watermarks['default'] = defaultdict(lambda: 0) + + self.check.check(self.instance) + + +if __name__ == '__main__': + unittest.main() From 644adaa1d00db6b4722d10bf952c1e2ebe447e11 Mon Sep 17 00:00:00 2001 From: Jesse Zoldak Date: Wed, 30 Jul 2014 11:40:45 -0400 Subject: [PATCH 2/5] Fix jenkins agent check when build does not yet have results --- checks.d/jenkins.py | 6 ++++++ tests/test_jenkins.py | 8 +++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/checks.d/jenkins.py b/checks.d/jenkins.py index 910421a6b4..e0265b0b5d 100644 --- a/checks.d/jenkins.py +++ b/checks.d/jenkins.py @@ -96,6 +96,12 @@ def _get_build_results(self, instance_key, job_dir): except Exception: continue + # If the metadata does not include the results, then + # the build is still running so do not process it yet. + build_result = build_metadata.get('result', None) + if build_result is None: + break + output = { 'job_name': job_name, 'timestamp': timestamp, diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 358d6d77cd..510e0ac907 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -6,7 +6,6 @@ import shutil import logging -from nose.tools import raises from tests.common import get_check logger = logging.getLogger(__file__) @@ -114,10 +113,6 @@ def testCheckCreatesEvents(self): assert 'jenkins.job.duration' in metrics assert len(metrics) == 2 - # This failure is expected due to a bug in the current code. - # TODO: fix the logic in jenkins.py and update this testcase with - # correct expectations. - @raises(KeyError) def testCheckWithRunningBuild(self): """ Test under the conditions of a jenkins build still running. @@ -131,6 +126,9 @@ def testCheckWithRunningBuild(self): self.check.check(self.instance) + # The check method does not return anything, so this testcase passes + # if the high_watermark was NOT updated and no exceptions were raised. + assert self.check.high_watermarks[self.instance['name']]['foo'] == 0 if __name__ == '__main__': unittest.main() From bb6d8405e42cc6e1a37b3256436ce36283cd2e43 Mon Sep 17 00:00:00 2001 From: Jesse Zoldak Date: Wed, 30 Jul 2014 17:25:25 -0400 Subject: [PATCH 3/5] Refactor jenkins agent tests --- tests/test_jenkins.py | 100 ++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 38 deletions(-) diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 510e0ac907..b39d73ab21 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -5,6 +5,7 @@ import tempfile import shutil import logging +import xml.etree.ElementTree as ET from tests.common import get_check @@ -13,19 +14,9 @@ DATETIME_FORMAT = '%Y-%m-%d_%H-%M-%S' LOG_DATA = 'Finished: SUCCESS' -BUILD_METADATA = """ - - 20783 - SUCCESS - 487 - -""" -NO_RESULT_YET_METADATA = """ - - 20783 - 487 - -""" +SUCCESSFUL_BUILD = {'number': '99', 'result': 'SUCCESS', 'duration': '60'} +NO_RESULTS_YET = {'number': '99', 'duration': '60'} +UNSUCCESSFUL_BUILD = {'number': '99', 'result': 'ABORTED', 'duration': '60'} CONFIG = """ init_config: @@ -35,57 +26,66 @@ jenkins_home: """ +def dict_to_xml(metadata_dict): + """ Convert a dict to xml for use in a build.xml file """ + build = ET.Element('build') + for k, v in metadata_dict.iteritems(): + node = ET.SubElement(build, k) + node.text = v + + return ET.tostring(build) + +def write_file(file_name, log_data): + with open(file_name, 'w') as log_file: + log_file.write(log_data) + class TestJenkins(unittest.TestCase): def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.config_yaml = CONFIG.replace('', self.tmp_dir) + self._create_old_build() def tearDown(self): # Clean up the temp directory shutil.rmtree(self.tmp_dir) - def _create_builds(self, metadata): + def _create_old_build(self): # As coded, the jenkins dd agent needs more than one result # in order to get the last valid build. - today = datetime.date.today() - yesterday = today - datetime.timedelta(days=1) - - old_date = yesterday.strftime(DATETIME_FORMAT) - todays_date = today.strftime(DATETIME_FORMAT) - - self._create_build(old_date, metadata) - self._create_build(todays_date, metadata) + # Create one for yesterday. + metadata = dict_to_xml(SUCCESSFUL_BUILD) + yesterday = datetime.date.today() - datetime.timedelta(days=1) + self._populate_build_dir(metadata, yesterday) def _create_check(self): # Create the jenkins check self.check, instances = get_check('jenkins', self.config_yaml) self.instance = instances[0] - def _create_build(self, datestring, metadata): + def _populate_build_dir(self, metadata, time=None): # The jenkins dd agent requires the build metadata file and a log file of results + time = time or datetime.datetime.now() + datestring = time.strftime(DATETIME_FORMAT) build_dir = os.path.join(self.tmp_dir, 'jobs', 'foo', 'builds', datestring) os.makedirs(build_dir) - metadata_file = open(os.path.join(build_dir, 'build.xml'), 'w+b') - log_file = open(os.path.join(build_dir, 'log'), 'w+b') - + log_file = os.path.join(build_dir, 'log') log_data = LOG_DATA - self._write_file(log_file, log_data) + write_file(log_file, log_data) + metadata_file = os.path.join(build_dir, 'build.xml') build_metadata = metadata - self._write_file(metadata_file, build_metadata) - - def _write_file(self, log_file, log_data): - log_file.write(log_data) - log_file.flush() + write_file(metadata_file, build_metadata) def testParseBuildLog(self): """ Test doing a jenkins check. This will parse the logs but since there was no previous high watermark no event will be created. """ - self._create_builds(BUILD_METADATA) + metadata = dict_to_xml(SUCCESSFUL_BUILD) + + self._populate_build_dir(metadata) self._create_check() self.check.check(self.instance) @@ -93,32 +93,56 @@ def testParseBuildLog(self): # if the high_watermark was set and no exceptions were raised. self.assertTrue(self.check.high_watermarks[self.instance['name']]['foo'] > 0) - def testCheckCreatesEvents(self): + def testCheckSuccessfulEvent(self): """ - Test that a successful build will create metrics to report in. + Test that a successful build will create the correct metrics. """ - self._create_builds(BUILD_METADATA) + metadata = dict_to_xml(SUCCESSFUL_BUILD) + + self._populate_build_dir(metadata) self._create_check() # Set the high_water mark so that the next check will create events self.check.high_watermarks['default'] = defaultdict(lambda: 0) - # Do a check self.check.check(self.instance) results = self.check.get_metrics() metrics = [r[0] for r in results] + assert len(metrics) == 2 assert 'jenkins.job.success' in metrics assert 'jenkins.job.duration' in metrics + + def testCheckUnsuccessfulEvent(self): + """ + Test that an unsuccessful build will create the correct metrics. + """ + metadata = dict_to_xml(UNSUCCESSFUL_BUILD) + + self._populate_build_dir(metadata) + self._create_check() + + # Set the high_water mark so that the next check will create events + self.check.high_watermarks['default'] = defaultdict(lambda: 0) + + self.check.check(self.instance) + + results = self.check.get_metrics() + metrics = [r[0] for r in results] + assert len(metrics) == 2 + assert 'jenkins.job.failure' in metrics + assert 'jenkins.job.duration' in metrics def testCheckWithRunningBuild(self): """ Test under the conditions of a jenkins build still running. The build.xml file will exist but it will not yet have a result. """ - self._create_builds(NO_RESULT_YET_METADATA) + metadata = dict_to_xml(NO_RESULTS_YET) + + self._populate_build_dir(metadata) self._create_check() # Set the high_water mark so that the next check will create events From a4c9c532deb5b2e5b10869b16ef02f3045b78cf4 Mon Sep 17 00:00:00 2001 From: Jesse Zoldak Date: Wed, 30 Jul 2014 19:19:02 -0400 Subject: [PATCH 4/5] Add build result as a tag to jenkins events --- checks.d/jenkins.py | 10 +++++++++- tests/test_jenkins.py | 32 ++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/checks.d/jenkins.py b/checks.d/jenkins.py index e0265b0b5d..a7d76f8b52 100644 --- a/checks.d/jenkins.py +++ b/checks.d/jenkins.py @@ -107,9 +107,13 @@ def _get_build_results(self, instance_key, job_dir): 'timestamp': timestamp, 'event_type': 'build result' } + output.update(build_metadata) self.high_watermarks[instance_key][job_name] = timestamp + self.log.debug("Processing %s results '%s'" % (job_name, output)) + yield output + # If it not a new build, stop here else: break @@ -146,7 +150,11 @@ def check(self, instance, create_event=True): self.log.debug("Creating event for job: %s" % output['job_name']) self.event(output) - tags = ['job_name:%s' % output['job_name']] + tags = [ + 'job_name:%s' % output['job_name'], + 'result:%s' % output['result'] + ] + if 'branch' in output: tags.append('branch:%s' % output['branch']) self.gauge("jenkins.job.duration", float(output['duration'])/1000.0, tags=tags) diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index b39d73ab21..c97c9d0414 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -107,12 +107,18 @@ def testCheckSuccessfulEvent(self): self.check.check(self.instance) - results = self.check.get_metrics() - metrics = [r[0] for r in results] + metrics = self.check.get_metrics() + + metrics_names = [m[0] for m in metrics] + assert len(metrics_names) == 2 + assert 'jenkins.job.success' in metrics_names + assert 'jenkins.job.duration' in metrics_names + + metrics_tags = [m[3] for m in metrics] + for tag in metrics_tags: + assert 'job_name:foo' in tag.get('tags') + assert 'result:SUCCESS' in tag.get('tags') - assert len(metrics) == 2 - assert 'jenkins.job.success' in metrics - assert 'jenkins.job.duration' in metrics def testCheckUnsuccessfulEvent(self): """ @@ -128,12 +134,18 @@ def testCheckUnsuccessfulEvent(self): self.check.check(self.instance) - results = self.check.get_metrics() - metrics = [r[0] for r in results] + metrics = self.check.get_metrics() + + metrics_names = [m[0] for m in metrics] + assert len(metrics_names) == 2 + assert 'jenkins.job.failure' in metrics_names + assert 'jenkins.job.duration' in metrics_names + + metrics_tags = [m[3] for m in metrics] + for tag in metrics_tags: + assert 'job_name:foo' in tag.get('tags') + assert 'result:ABORTED' in tag.get('tags') - assert len(metrics) == 2 - assert 'jenkins.job.failure' in metrics - assert 'jenkins.job.duration' in metrics def testCheckWithRunningBuild(self): """ From fd6313f719ac78017110644c34c2fa5986c20240 Mon Sep 17 00:00:00 2001 From: Jesse Zoldak Date: Thu, 31 Jul 2014 13:20:02 -0400 Subject: [PATCH 5/5] Add build number to tags for jenkins results --- checks.d/jenkins.py | 3 ++- tests/test_jenkins.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/checks.d/jenkins.py b/checks.d/jenkins.py index a7d76f8b52..15a0359cb4 100644 --- a/checks.d/jenkins.py +++ b/checks.d/jenkins.py @@ -152,7 +152,8 @@ def check(self, instance, create_event=True): tags = [ 'job_name:%s' % output['job_name'], - 'result:%s' % output['result'] + 'result:%s' % output['result'], + 'build_number:%s' % output['number'] ] if 'branch' in output: diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index c97c9d0414..a116d240e1 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -118,6 +118,7 @@ def testCheckSuccessfulEvent(self): for tag in metrics_tags: assert 'job_name:foo' in tag.get('tags') assert 'result:SUCCESS' in tag.get('tags') + assert 'build_number:99' in tag.get('tags') def testCheckUnsuccessfulEvent(self): @@ -145,6 +146,7 @@ def testCheckUnsuccessfulEvent(self): for tag in metrics_tags: assert 'job_name:foo' in tag.get('tags') assert 'result:ABORTED' in tag.get('tags') + assert 'build_number:99' in tag.get('tags') def testCheckWithRunningBuild(self):