From 4b3314326e68c0be334fdcdf1a92bb7c3c830ab4 Mon Sep 17 00:00:00 2001 From: binux Date: Tue, 11 Nov 2014 21:07:43 +0800 Subject: [PATCH] improve test coverage --- .coveragerc | 14 ++++++ .travis.yml | 2 +- database/base/resultdb.py | 10 ++--- test/test_processor.py | 23 ++++++++++ test/test_result_worker.py | 87 ++++++++++++++++++++++++++++++++++++++ test/test_scheduler.py | 21 +++++++++ 6 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 test/test_result_worker.py diff --git a/.coveragerc b/.coveragerc index 451c04f23..642dd51c6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,17 @@ +[run] +source = + database + scheduler + fetcher + processor + result + libs + [report] +omit = + libs/sample_handler.py + libs/pprint.py + exclude_lines = pragma: no cover def __repr__ @@ -9,3 +22,4 @@ exclude_lines = if 0: if __name__ == .__main__.: except ImportError: + pass diff --git a/.travis.yml b/.travis.yml index 9fb0f850a..82146f341 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: - pip install --allow-all-external -r requirements.txt - pip install coveralls script: - coverage run --source=database,scheduler,fetcher,processor,result,libs --omit=libs/sample_handler.py,libs/pprint.py ./runtest.py + coverage run ./runtest.py after_success: - coverage combine - coveralls diff --git a/database/base/resultdb.py b/database/base/resultdb.py index ce3658260..e1d63d0b0 100644 --- a/database/base/resultdb.py +++ b/database/base/resultdb.py @@ -22,16 +22,16 @@ class ResultDB(object): """ def save(self, project, taskid, url, result): - pass + raise NotImplementedError def select(self, project, fields=None, offset=0, limit=None): - pass + raise NotImplementedError def count(self, project): - pass + raise NotImplementedError def get(self, project, taskid, fields=None): - pass + raise NotImplementedError def drop(self, project): - pass + raise NotImplementedError diff --git a/test/test_processor.py b/test/test_processor.py index 5de6ba9fd..0f8461162 100644 --- a/test/test_processor.py +++ b/test/test_processor.py @@ -259,4 +259,27 @@ def test_30_new_task(self): self.in_queue.put((task, fetch_result)) time.sleep(1) self.assertFalse(self.status_queue.empty()) + while not self.status_queue.empty(): + self.status_queue.get() self.assertFalse(self.newtask_queue.empty()) + + def test_40_index_page(self): + task = None + while not self.newtask_queue.empty(): + task = self.newtask_queue.get() + self.assertIsNotNone(task) + + fetch_result = { + "orig_url": task['url'], + "content": "binux", + "headers": {}, + "status_code": 200, + "url": task['url'], + "time": 0, + } + self.in_queue.put((task, fetch_result)) + time.sleep(1) + self.assertFalse(self.status_queue.empty()) + self.assertFalse(self.newtask_queue.empty()) + task = self.newtask_queue.get() + self.assertEqual(task['url'], 'http://binux.me/') diff --git a/test/test_result_worker.py b/test/test_result_worker.py new file mode 100644 index 000000000..e3fba4a20 --- /dev/null +++ b/test/test_result_worker.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8: +# Author: Binux +# http://binux.me +# Created on 2014-11-11 20:52:53 + +import os +import time +import unittest2 as unittest +import logging.config +logging.config.fileConfig("logging.conf") + +import shutil +from multiprocessing import Queue +from database.sqlite import resultdb +from result.result_worker import ResultWorker +from libs.utils import run_in_subprocess, run_in_thread +class TestProcessor(unittest.TestCase): + resultdb_path = './test/data/result.db' + + @classmethod + def setUpClass(self): + shutil.rmtree('./test/data/', ignore_errors=True) + os.makedirs('./test/data/') + + def get_resultdb(): + return resultdb.ResultDB(self.resultdb_path) + self.resultdb = get_resultdb() + self.inqueue = Queue(10) + + def run_result_worker(): + self.result_worker = ResultWorker(get_resultdb(), self.inqueue) + self.result_worker.run() + self.process = run_in_thread(run_result_worker) + time.sleep(1) + + @classmethod + def tearDownClass(self): + if self.process.is_alive(): + self.result_worker.quit() + self.process.join(2) + assert not self.process.is_alive() + shutil.rmtree('./test/data/', ignore_errors=True) + + def test_10_bad_result(self): + self.inqueue.put(({'project': 'test_project'}, {})) + self.resultdb._list_project() + self.assertEqual(len(self.resultdb.projects), 0) + self.assertEqual(self.resultdb.count('test_project'), 0) + + def test_20_insert_result(self): + data = { + 'a': 'b' + } + self.inqueue.put(({ + 'project': 'test_project', + 'taskid': 'id1', + 'url': 'url1' + }, data)) + time.sleep(0.1) + self.resultdb._list_project() + self.assertEqual(len(self.resultdb.projects), 1) + self.assertEqual(self.resultdb.count('test_project'), 1) + + result = self.resultdb.get('test_project', 'id1') + self.assertEqual(result['result'], data) + + def test_30_overwrite(self): + self.inqueue.put(({ + 'project': 'test_project', + 'taskid': 'id1', + 'url': 'url1' + }, "abc")) + time.sleep(0.1) + result = self.resultdb.get('test_project', 'id1') + self.assertEqual(result['result'], "abc") + + def test_40_insert_list(self): + self.inqueue.put(({ + 'project': 'test_project', + 'taskid': 'id2', + 'url': 'url1' + }, ['a', 'b'])) + time.sleep(0.1) + result = self.resultdb.get('test_project', 'id2') + self.assertEqual(result['result'], ['a', 'b']) diff --git a/test/test_scheduler.py b/test/test_scheduler.py index 08353d171..457c06418 100644 --- a/test/test_scheduler.py +++ b/test/test_scheduler.py @@ -128,6 +128,7 @@ def test_10_new_task_ignore(self): 'url': 'url' }) self.assertEqual(self.rpc.size(), 0) + self.assertEqual(len(self.rpc.get_active_tasks()), 0) def test_20_new_project(self): self.projectdb.insert('test_project', { @@ -145,6 +146,7 @@ def test_30_update_project(self): with self.assertRaises(Queue.Empty): task = self.scheduler2fetcher.get(timeout=0.1) self.projectdb.update('test_project', status="DEBUG") + self.rpc.update_project() task = self.scheduler2fetcher.get(timeout=5) self.assertIsNotNone(task) @@ -173,6 +175,7 @@ def test_35_new_task(self): self.assertEqual(self.rpc.counter('all', 'sum')['test_project']['pending'], 1) task = self.scheduler2fetcher.get(timeout=5) + self.assertEqual(len(self.rpc.get_active_tasks()), 1) self.assertIsNotNone(task) self.assertEqual(task['project'], 'test_project') self.assertIn('fetch', task) @@ -258,6 +261,24 @@ def test_80_newtask_age_ignore(self): time.sleep(0.1) self.assertEqual(self.rpc.size(), 0) + def test_82_newtask_via_rpc(self): + self.rpc.newtask({ + 'taskid': 'taskid', + 'project': 'test_project', + 'url': 'url', + 'fetch': { + 'data': 'abc', + }, + 'process': { + 'data': 'abc', + }, + 'schedule': { + 'age': 30, + }, + }) + time.sleep(0.1) + self.assertEqual(self.rpc.size(), 0) + def test_90_newtask_with_itag(self): time.sleep(0.1) self.newtask_queue.put({