API: Add function to start the daemon

So far, the daemon could only be started through `verdi` and there was
no way to do it from the Python API. Here we add the `start_daemon`
method to the `aiida.engine.daemon.client.DaemonClient` class which will
start the daemon when called.

To start the daemon, the function will actually still invoke the `verdi`
command through a subprocess. The reason is that starting the daemon
will put the calling process in the background, which is not the desired
behavior when calling it from the API. The actual code that launches the
circus daemon is moved from the `verdi` command to the `_start_daemon`
method of the `DaemonClient`. In this way, the daemon functionality is
more self-contained. By making it a protected method, we signal that
users of the Python API should probably not use it, but use the public
`start_daemon` instead.

This implementation may seem to have some circularity, as `start_daemon`
will call a `verdi` command, which in turn will call the `_start_daemon`
method of the `DaemonClient`. The reason for this is that the `verdi`
command ensures that the correct profile is loaded before starting the
daemon. We could make a separate CLI end point independent of `verdi`
that just serves to load a profile and start the daemon, but that seems
unnecessarily complicated at this point.

Besides the added function, which was the main goal, the code is also
refactored considerably. The implementation of the command line command
`verdi daemon start-circus` is now moved to the `_start_circus` method
of the `DaemonClient` class. The `verdi devel run_daemon` command is
moved to `verdi daemon worker`, which makes more sense as a location.
This command launches a single daemon worker, which is nothing more than
an AiiDA process that runs a `Runner` instance in blocking mode. The
`verdi daemon start` will run a circus daemon that manages instances of
these daemon workers. To better match the nomenclature, the module
`aiida.engine.daemon.runner` was renamed to `worker`.

aiida/cmdline/commands/cmd_daemon.py

@@ -8,8 +8,6 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """`verdi daemon` commands."""
-
-import os
 import subprocess
 import sys
 import time
@@ -233,7 +231,6 @@ def restart(ctx, reset, no_wait):
     from aiida.engine.daemon.client import get_daemon_client
 
     client = get_daemon_client()
-
     wait = not no_wait
 
     if reset:
@@ -271,84 +268,13 @@ def start_circus(foreground, number):
 
     .. note:: this should not be called directly from the commandline!
     """
-    from circus import get_arbiter
-    from circus import logger as circus_logger
-    from circus.circusd import daemonize
-    from circus.pidfile import Pidfile
-    from circus.util import check_future_exception_and_log, configure_logger
-
     from aiida.engine.daemon.client import get_daemon_client
+    get_daemon_client()._start_daemon(number_workers=number, foreground=foreground)  # pylint: disable=protected-access
 
-    if foreground and number > 1:
-        raise click.ClickException('can only run a single worker when running in the foreground')
-
-    client = get_daemon_client()
-
-    loglevel = client.loglevel
-    logoutput = '-'
-
-    if not foreground:
-        logoutput = client.circus_log_file
-
-    arbiter_config = {
-        'controller': client.get_controller_endpoint(),
-        'pubsub_endpoint': client.get_pubsub_endpoint(),
-        'stats_endpoint': client.get_stats_endpoint(),
-        'logoutput': logoutput,
-        'loglevel': loglevel,
-        'debug': False,
-        'statsd': True,
-        'pidfile': client.circus_pid_file,
-        'watchers': [{
-            'cmd': client.cmd_string,
-            'name': client.daemon_name,
-            'numprocesses': number,
-            'virtualenv': client.virtualenv,
-            'copy_env': True,
-            'stdout_stream': {
-                'class': 'FileStream',
-                'filename': client.daemon_log_file,
-            },
-            'stderr_stream': {
-                'class': 'FileStream',
-                'filename': client.daemon_log_file,
-            },
-            'env': get_env_with_venv_bin(),
-        }]
-    }  # yapf: disable
-
-    if not foreground:
-        daemonize()
-
-    arbiter = get_arbiter(**arbiter_config)
-    pidfile = Pidfile(arbiter.pidfile)
 
-    try:
-        pidfile.create(os.getpid())
-    except RuntimeError as exception:
-        echo.echo_critical(str(exception))
-
-    # Configure the logger
-    loggerconfig = None
-    loggerconfig = loggerconfig or arbiter.loggerconfig or None
-    configure_logger(circus_logger, loglevel, logoutput, loggerconfig)
-
-    # Main loop
-    should_restart = True
-
-    while should_restart:
-        try:
-            future = arbiter.start()
-            should_restart = False
-            if check_future_exception_and_log(future) is None:
-                should_restart = arbiter._restarting  # pylint: disable=protected-access
-        except Exception as exception:
-            # Emergency stop
-            arbiter.loop.run_sync(arbiter._emergency_stop)  # pylint: disable=protected-access
-            raise exception
-        except KeyboardInterrupt:
-            pass
-        finally:
-            arbiter = None
-            if pidfile is not None:
-                pidfile.unlink()
+@verdi_daemon.command('worker')
+@decorators.with_dbenv()
+def worker():
+    """Run a single daemon worker in the current interpreter."""
+    from aiida.engine.daemon.worker import start_daemon_worker
+    start_daemon_worker()

aiida/cmdline/commands/cmd_devel.py

@@ -73,14 +73,6 @@ def devel_check_undesired_imports():
     echo.echo_success('no issues detected')
 
 
-@verdi_devel.command('run_daemon')
-@decorators.with_dbenv()
-def devel_run_daemon():
-    """Run a daemon instance in the current interpreter."""
-    from aiida.engine.daemon.runner import start_daemon
-    start_daemon()
-
-
 @verdi_devel.command('validate-plugins')
 @decorators.with_dbenv()
 def devel_validate_plugins():