[WIP] plugin fixtures: return testrunner result (#1676)

* plugin fixtures: return testrunner result

This is needed in order e.g. to fail on travis, when tests don't pass.

* set backend before creating fixture manager

backend was being set too late (as a matter of fact, all tests
using the fixture manager have, so far, tested only the django backend).

.travis-data/test_plugin_testcase.py

@@ -108,4 +108,7 @@ def test_stupid(self):
 if __name__ == '__main__':
     MODULE = sys.modules[__name__]
     SUITE = unittest.defaultTestLoader.loadTestsFromModule(MODULE)
-    TestRunner().run(SUITE, backend=determine_backend())
+    RESULT = TestRunner().run(SUITE, backend=determine_backend())
+
+    EXIT_CODE = int(not RESULT.wasSuccessful())
+    sys.exit(EXIT_CODE)

aiida/utils/fixtures.py

@@ -286,6 +286,10 @@ def backend(self):
 
     @backend.setter
     def backend(self, backend):
+        if self.__is_running_on_test_profile:
+            raise FixtureError(
+                'backend cannot be changed after setting up the environment')
+
         valid_backends = [BACKEND_DJANGO, BACKEND_SQLA]
         if backend not in valid_backends:
             raise ValueError('invalid backend {}, must be one of {}'.format(
@@ -387,24 +391,27 @@ def has_profile_open(self):
 
 
 @contextmanager
-def fixture_manager():
+def fixture_manager(backend=BACKEND_DJANGO):
     """
     Context manager for FixtureManager objects
 
     Example test runner (unittest)::
 
-        with fixture_manager() as fixture_mgr:
+        with fixture_manager(backend) as fixture_mgr:
             # ready for tests
         # everything cleaned up
 
     Example fixture (pytest)::
 
         def aiida_profile():
-            with fixture_manager() as fixture_mgr:
+            with fixture_manager(backend) as fixture_mgr:
                 yield fixture_mgr
+
+    :param backend: database backend, either BACKEND_SQLA or BACKEND_DJANGO
     """
     try:
         if not _GLOBAL_FIXTURE_MANAGER.has_profile_open():
+            _GLOBAL_FIXTURE_MANAGER.backend = backend
             _GLOBAL_FIXTURE_MANAGER.create_profile()
         yield _GLOBAL_FIXTURE_MANAGER
     finally:
@@ -467,6 +474,5 @@ def run(self, suite, backend=BACKEND_DJANGO):
         """
         from aiida.utils.capturing import Capturing
         with Capturing():
-            with fixture_manager() as manager:
-                manager.backend = backend
-                super(TestRunner, self).run(suite)
+            with fixture_manager(backend=backend):
+                return super(TestRunner, self).run(suite)