diff --git a/etc/pyca.conf b/etc/pyca.conf index 5124eda3..f4c85a82 100644 --- a/etc/pyca.conf +++ b/etc/pyca.conf @@ -278,3 +278,19 @@ password = 'CHANGE_ME' # Log format configuration # Default: [%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s #format = [%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s + + +[http] + +# Number of seconds after which non-upload HTTP requests will time out. +# Note that this is the total request duration including the establishment of the connection. Also see +# connection_timeout to set a timeout for the latter. Setting this to 0 will disable a time out for HTTP requests. +# HTTP requests that ingest files will not time out. +# Default: 300 +#timeout = 300 + +# Number of seconds after which the connection establishment of HTTP requests will time out. +# This applies to all HTTP requests pyCA will make including upload requests. The connection is considered established +# after the TCP, TLS or QUIC handshakes. +# Default: 180 +#connection_timeout = 180 diff --git a/pyca/config.py b/pyca/config.py index 959abe16..43b3340b 100644 --- a/pyca/config.py +++ b/pyca/config.py @@ -64,6 +64,10 @@ level = option('debug', 'info', 'warning', 'error', default='info') format = string(default='[%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s') +[http] +timeout = integer(min=0, default=300) +connection_timeout = integer(min=0, default=180) + [services] ''' # noqa diff --git a/pyca/ingest.py b/pyca/ingest.py index 5e984062..6b585f2f 100644 --- a/pyca/ingest.py +++ b/pyca/ingest.py @@ -59,7 +59,7 @@ def ingest(event): # create mediapackage logger.info('Creating new mediapackage') - mediapackage = http_request(service_url + '/createMediaPackage') + mediapackage = http_request(service_url + '/createMediaPackage', timeout=0) # extract workflow_def, workflow_config and add DC catalogs prop = 'org.opencastproject.capture.agent.properties' @@ -78,7 +78,8 @@ def ingest(event): fields = [('mediaPackage', mediapackage), ('flavor', 'dublincore/%s' % name), ('dublinCore', data.encode('utf-8'))] - mediapackage = http_request(service_url + '/addDCCatalog', fields) + mediapackage = http_request(service_url + '/addDCCatalog', fields, + timeout=0) else: logger.info('Not uploading %s', attachment.get('x-apple-filename')) @@ -90,7 +91,8 @@ def ingest(event): track = track.encode('ascii', 'ignore') fields = [('mediaPackage', mediapackage), ('flavor', flavor), ('BODY1', (pycurl.FORM_FILE, track))] - mediapackage = http_request(service_url + '/addTrack', fields) + mediapackage = http_request(service_url + '/addTrack', fields, + timeout=0) # ingest logger.info('Ingest recording') @@ -101,7 +103,7 @@ def ingest(event): fields.append(('workflowInstanceId', event.uid.encode('ascii', 'ignore'))) fields += workflow_config - mediapackage = http_request(service_url + '/ingest', fields) + mediapackage = http_request(service_url + '/ingest', fields, timeout=0) # Update status recording_state(event.uid, 'upload_finished') diff --git a/pyca/ui/opencast_commands.py b/pyca/ui/opencast_commands.py index a9cebdb1..573c9571 100644 --- a/pyca/ui/opencast_commands.py +++ b/pyca/ui/opencast_commands.py @@ -53,7 +53,7 @@ def schedule(title='pyCA Recording', duration=60, creator=None): # create media package logger.info('Creating new media package') - mediapackage = http_request(service_url + '/createMediaPackage') + mediapackage = http_request(service_url + '/createMediaPackage', timeout=0) # add dublin core catalog start = datetime.utcnow() + timedelta(seconds=10) @@ -68,12 +68,13 @@ def schedule(title='pyCA Recording', duration=60, creator=None): fields = [('mediaPackage', mediapackage), ('flavor', 'dublincore/episode'), ('dublinCore', dublincore)] - mediapackage = http_request(service_url + '/addDCCatalog', fields) + mediapackage = http_request(service_url + '/addDCCatalog', fields, + timeout=0) # schedule event logger.info('Scheduling recording') fields = [('mediaPackage', mediapackage)] - mediapackage = http_request(service_url + '/schedule', fields) + mediapackage = http_request(service_url + '/schedule', fields, timeout=0) # Update status logger.info('Event successfully scheduled') diff --git a/pyca/utils.py b/pyca/utils.py index e7783d19..56169419 100644 --- a/pyca/utils.py +++ b/pyca/utils.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -def http_request(url, post_data=None): +def http_request(url, post_data=None, timeout=None): '''Make an HTTP request to a given URL with optional parameters. ''' logger.debug('Requesting URL: %s', url) @@ -54,6 +54,12 @@ def http_request(url, post_data=None): curl.MAX_SEND_SPEED_LARGE, config('ingest', 'upload_rate')) + curl.setopt(curl.CONNECTTIMEOUT, config('http', 'connection_timeout')) + if timeout is not None: + curl.setopt(curl.TIMEOUT, timeout) + else: + curl.setopt(curl.TIMEOUT, config('http', 'timeout')) + if post_data: curl.setopt(curl.HTTPPOST, post_data) curl.setopt(curl.WRITEFUNCTION, buf.write) diff --git a/tests/test_agentstate.py b/tests/test_agentstate.py index f4a5e4df..0f56a336 100644 --- a/tests/test_agentstate.py +++ b/tests/test_agentstate.py @@ -15,7 +15,7 @@ class TestPycaAgentState(unittest.TestCase): def setUp(self): - utils.http_request = lambda x, y=False: b'xxx' + utils.http_request = lambda x, y=False, timeout=0: b'xxx' self.fd, self.dbfile = tempfile.mkstemp() config.config()['agent']['database'] = 'sqlite:///' + self.dbfile config.config()['service-capture.admin'] = [''] diff --git a/tests/test_capture.py b/tests/test_capture.py index fa2193e8..5ad6560b 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -17,7 +17,7 @@ class TestPycaCapture(unittest.TestCase): def setUp(self): - utils.http_request = lambda x, y=False: b'xxx' + utils.http_request = lambda x, y=False, timeout=0: b'xxx' self.fd, self.dbfile = tempfile.mkstemp() self.cadir = tempfile.mkdtemp() preview = os.path.join(self.cadir, 'preview.png') diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 9a7966ef..6dab4008 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -19,7 +19,7 @@ class TestPycaIngest(unittest.TestCase): def setUp(self): - ingest.http_request = lambda x, y=False: b'xxx' + ingest.http_request = lambda x, y=False, timeout=0: b'xxx' self.fd, self.dbfile = tempfile.mkstemp() self.cadir = tempfile.mkdtemp() config.config('agent')['database'] = 'sqlite:///' + self.dbfile diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 542c58cc..6452f6a9 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -29,7 +29,7 @@ class TestPycaCapture(unittest.TestCase): END:VCALENDAR''' % END).replace('\n ', '\r\n').encode('utf-8') def setUp(self): - utils.http_request = lambda x, y=False: b'xxx' + utils.http_request = lambda x, y=False, timeout=0: b'xxx' self.fd, self.dbfile = tempfile.mkstemp() config.config()['agent']['database'] = 'sqlite:///' + self.dbfile config.config()['services']['org.opencastproject.scheduler'] = [''] diff --git a/tests/test_ui_opencast_commands.py b/tests/test_ui_opencast_commands.py index 730c4e9f..19df300e 100644 --- a/tests/test_ui_opencast_commands.py +++ b/tests/test_ui_opencast_commands.py @@ -12,7 +12,7 @@ class TestPycaIngest(unittest.TestCase): def setUp(self): - opencast_commands.http_request = lambda x, y=False: b'xxx' + opencast_commands.http_request = lambda x, y=False, timeout=0: b'xxx' opencast_commands.service = lambda x, force_update=False: [''] def test_schedule_defaults(self): diff --git a/tests/test_utils.py b/tests/test_utils.py index 24717a59..23aff295 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -44,7 +44,7 @@ def test_get_service(self): "error_state_trigger":0, "warning_state_trigger":0}}}'''.encode('utf-8') # Mock http_request method - utils.http_request = lambda x, y=False: res + utils.http_request = lambda x, y=False, timeout=0: res endpoint = u'https://octestallinone.virtuos.uos.de/capture-admin' self.assertEqual(utils.get_service(''), [endpoint]) @@ -75,7 +75,7 @@ def test_http_request_mocked_curl(self): reload(utils.pycurl) def test_register_ca(self): - utils.http_request = lambda x, y=False: b'xxx' + utils.http_request = lambda x, y=False, timeout=0: b'xxx' utils.register_ca() utils.http_request = should_fail utils.register_ca() @@ -83,7 +83,7 @@ def test_register_ca(self): utils.register_ca() def test_recording_state(self): - utils.http_request = lambda x, y=False: b'' + utils.http_request = lambda x, y=False, timeout=0: b'' utils.recording_state('123', 'recording') utils.http_request = should_fail utils.recording_state('123', 'recording') @@ -91,7 +91,7 @@ def test_recording_state(self): utils.recording_state('123', 'recording') def test_set_service_status_immediate(self): - utils.http_request = lambda x, y=False: b'' + utils.http_request = lambda x, y=False, timeout=0: b'' utils.set_service_status_immediate(db.Service.SCHEDULE, db.ServiceStatus.IDLE) utils.set_service_status_immediate(db.Service.INGEST, diff --git a/tests/tools.py b/tests/tools.py index bb8a9b8a..bfdb29b7 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -50,6 +50,8 @@ class CurlMock(): FAILONERROR = 11 FOLLOWLOCATION = 12 MAX_SEND_SPEED_LARGE = 13 + CONNECTTIMEOUT = 14 + TIMEOUT = 15 def setopt(self, *args): pass