[#2977] Basic implementation of background jobs via python-rq

ckan/config/deployment.ini_tmpl

@@ -79,6 +79,12 @@ ckan.site_id = default
 #solr_url = http://127.0.0.1:8983/solr
 
 
+## Redis Settings
+
+# URL to your Redis instance, including the database to be used.
+#ckan.redis.url = redis://localhost:6379/0
+
+
 ## CORS Settings
 
 # If cors.origin_allow_all is true, all origins are allowed.

ckan/config/environment.py

@@ -16,6 +16,7 @@
 import ckan.plugins as p
 import ckan.lib.helpers as helpers
 import ckan.lib.app_globals as app_globals
+from ckan.lib.redis import is_redis_available
 import ckan.lib.render as render
 import ckan.lib.search as search
 import ckan.logic as logic
@@ -93,6 +94,10 @@ def find_controller(self, controller):
     for msg in msgs:
         warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
 
+    # Check Redis availability
+    if not is_redis_available():
+        log.critical('Could not connect to Redis.')
+
     # load all CKAN plugins
     p.load_all()
 
@@ -105,6 +110,7 @@ def find_controller(self, controller):
     'sqlalchemy.url': 'CKAN_SQLALCHEMY_URL',
     'ckan.datastore.write_url': 'CKAN_DATASTORE_WRITE_URL',
     'ckan.datastore.read_url': 'CKAN_DATASTORE_READ_URL',
+    'ckan.redis.url': 'CKAN_REDIS_URL',
     'solr_url': 'CKAN_SOLR_URL',
     'ckan.site_id': 'CKAN_SITE_ID',
     'ckan.site_url': 'CKAN_SITE_URL',

ckan/config/supervisor-ckan-worker.conf

@@ -0,0 +1,43 @@
+; =======================================================
+; Supervisor configuration for CKAN background job worker
+; =======================================================
+
+; 1. Copy this file to /etc/supervisr/conf.d
+; 2. Make sure the paths below match your setup
+
+
+[program:ckan-worker]
+
+; Use the full paths to the virtualenv and your configuration file here.
+command=/usr/lib/ckan/default/bin/paster --plugin=ckan jobs worker --config=/etc/ckan/default/production.ini
+
+
+; User the worker runs as.
+user=www-data
+
+
+; Start just a single worker. Increase this number if you have many or
+; particularly long running background jobs.
+numprocs=1
+process_name=%(program_name)s-%(process_num)02d
+
+
+; Log files.
+stdout_logfile=/var/log/ckan-worker.log
+stderr_logfile=/var/log/ckan-worker.log
+
+
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+
+
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
+

ckan/lib/celery_app.py

@@ -1,5 +1,11 @@
 # encoding: utf-8
 
+'''
+Celery background tasks management.
+
+This module is DEPRECATED, use ``ckan.lib.jobs`` instead.
+'''
+
 import ConfigParser
 import os
 import logging
@@ -9,6 +15,8 @@
 
 log = logging.getLogger(__name__)
 
+log.warning('ckan.lib.celery_app is deprecated, use ckan.lib.jobs instead.')
+
 LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES""".split()
 
 from celery import Celery