1038 organization hierarchy

ckan/config/deployment.ini_tmpl

@@ -67,6 +67,7 @@ ckan.auth.user_delete_groups = true
 ckan.auth.user_delete_organizations = true
 ckan.auth.create_user_via_api = false
 ckan.auth.create_user_via_web = true
+ckan.auth.roles_that_cascade_to_sub_groups = admin
 
 
 ## Search Settings

ckan/config/environment.py

@@ -367,8 +367,10 @@ def template_loaded(template):
     # Here we create the site user if they are not already in the database
     try:
         logic.get_action('get_site_user')({'ignore_auth': True}, None)
-    except sqlalchemy.exc.ProgrammingError:
-        # The database is not initialised.  This is a bit dirty.
+    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
+        # (ProgrammingError for Postgres, OperationalError for SQLite)
+        # The database is not initialised.  This is a bit dirty.  This occurs
+        # when running tests.
         pass
     except sqlalchemy.exc.InternalError:
         # The database is not initialised.  Travis hits this

ckan/lib/cli.py

@@ -1322,7 +1322,7 @@ class CreateTestDataCommand(CkanCommand):
                                     translations of terms
     create-test-data vocabs       - annakerenina, warandpeace, and some test
                                     vocabularies
-
+    create-test-data hierarchy    - hierarchy of groups
     '''
     summary = __doc__.split('\n')[0]
     usage = __doc__
@@ -1332,6 +1332,7 @@ class CreateTestDataCommand(CkanCommand):
     def command(self):
         self._load_config()
         self._setup_app()
+        from ckan import plugins
         from create_test_data import CreateTestData
 
         if self.args:
@@ -1356,6 +1357,8 @@ def command(self):
             CreateTestData.create_translations_test_data()
         elif cmd == 'vocabs':
             CreateTestData.create_vocabs_test_data()
+        elif cmd == 'hierarchy':
+            CreateTestData.create_group_hierarchy_test_data()
         else:
             print 'Command %s not recognized' % cmd
             raise NotImplementedError

ckan/lib/create_test_data.py

@@ -38,6 +38,12 @@ def create_family_test_data(cls, extra_users=[]):
                               relationships=family_relationships,
                               extra_user_names=extra_users)
 
+    @classmethod
+    def create_group_hierarchy_test_data(cls, extra_users=[]):
+        cls.create_users(group_hierarchy_users)
+        cls.create_groups(group_hierarchy_groups)
+        cls.create_arbitrary(group_hierarchy_datasets)
+
     @classmethod
     def create_test_user(cls):
         tester = model.User.by_name(u'tester')
@@ -215,18 +221,26 @@ def create_arbitrary(cls, package_dicts, relationships=[],
                             group = model.Group.by_name(unicode(group_name))
                             if not group:
                                 if not group_name in new_groups:
-                                    group = model.Group(name=unicode(group_name))
+                                    group = model.Group(name=
+                                                        unicode(group_name))
                                     model.Session.add(group)
                                     new_group_names.add(group_name)
                                     new_groups[group_name] = group
                                 else:
-                                    # If adding multiple packages with the same group name,
-                                    # model.Group.by_name will not find the group as the
-                                    # session has not yet been committed at this point.
-                                    # Fetch from the new_groups dict instead.
+                                    # If adding multiple packages with the same
+                                    # group name, model.Group.by_name will not
+                                    # find the group as the session has not yet
+                                    # been committed at this point.  Fetch from
+                                    # the new_groups dict instead.
                                     group = new_groups[group_name]
-                            member = model.Member(group=group, table_id=pkg.id, table_name='package')
+                            capacity = 'organization' if group.is_organization\
+                                       else 'public'
+                            member = model.Member(group=group, table_id=pkg.id,
+                                                  table_name='package',
+                                                  capacity=capacity)
                             model.Session.add(member)
+                            if group.is_organization:
+                                pkg.owner_org = group.id
                     elif attr == 'license':
                         pkg.license_id = val
                     elif attr == 'license_id':
@@ -315,38 +329,67 @@ def pkg(pkg_name):
     @classmethod
     def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""):
         '''A more featured interface for creating groups.
-        All group fields can be filled, packages added and they can
-        have an admin user.'''
+        All group fields can be filled, packages added, can have
+        an admin user and be a member of other groups.'''
         rev = model.repo.new_revision()
-        # same name as user we create below
         rev.author = cls.author
         if admin_user_name:
             admin_users = [model.User.by_name(admin_user_name)]
         else:
             admin_users = []
         assert isinstance(group_dicts, (list, tuple))
-        group_attributes = set(('name', 'title', 'description', 'parent_id'))
+        group_attributes = set(('name', 'title', 'description', 'parent_id',
+                                'type', 'is_organization'))
         for group_dict in group_dicts:
-            if model.Group.by_name(group_dict['name']):
-                log.warning('Cannot create group "%s" as it already exists.' % \
-                                (group_dict['name']))
+            if model.Group.by_name(unicode(group_dict['name'])):
+                log.warning('Cannot create group "%s" as it already exists.' %
+                            group_dict['name'])
                 continue
             pkg_names = group_dict.pop('packages', [])
             group = model.Group(name=unicode(group_dict['name']))
             group.type = auth_profile or 'group'
             for key in group_dict:
                 if key in group_attributes:
                     setattr(group, key, group_dict[key])
-                else:
+                elif key not in ('admins', 'editors', 'parent'):
                     group.extras[key] = group_dict[key]
             assert isinstance(pkg_names, (list, tuple))
             for pkg_name in pkg_names:
                 pkg = model.Package.by_name(unicode(pkg_name))
                 assert pkg, pkg_name
-                member = model.Member(group=group, table_id=pkg.id, table_name='package')
+                member = model.Member(group=group, table_id=pkg.id,
+                                      table_name='package')
                 model.Session.add(member)
             model.Session.add(group)
-            model.setup_default_user_roles(group, admin_users)
+            admins = [model.User.by_name(user_name)
+                      for user_name in group_dict.get('admins', [])] + \
+                     admin_users
+            for admin in admins:
+                member = model.Member(group=group, table_id=admin.id,
+                                      table_name='user', capacity='admin')
+                model.Session.add(member)
+            editors = [model.User.by_name(user_name)
+                       for user_name in group_dict.get('editors', [])]
+            for editor in editors:
+                member = model.Member(group=group, table_id=editor.id,
+                                      table_name='user', capacity='editor')
+                model.Session.add(member)
+            # Need to commit the current Group for two reasons:
+            # 1. It might have a parent, and the Member will need the Group.id
+            #    value allocated on commit.
+            # 2. The next Group created may have this Group as a parent so
+            #    creation of the Member needs to refer to this one.
+            model.Session.commit()
+            rev = model.repo.new_revision()
+            rev.author = cls.author
+            # add it to a parent's group
+            if 'parent' in group_dict:
+                parent = model.Group.by_name(unicode(group_dict['parent']))
+                assert parent, group_dict['parent']
+                member = model.Member(group=group, table_id=parent.id,
+                                      table_name='group', capacity='parent')
+                model.Session.add(member)
+            #model.setup_default_user_roles(group, admin_users)
             cls.group_names.add(group_dict['name'])
         model.repo.commit_and_remove()
 
@@ -362,7 +405,8 @@ def create(cls, auth_profile="", package_type=None):
  * Associated tags, etc etc
 '''
         if auth_profile == "publisher":
-            organization_group = model.Group(name=u"organization_group", type="organization")
+            organization_group = model.Group(name=u"organization_group",
+                                             type="organization")
 
         cls.pkg_names = [u'annakarenina', u'warandpeace']
         pkg1 = model.Package(name=cls.pkg_names[0], type=package_type)
@@ -483,11 +527,12 @@ def create(cls, auth_profile="", package_type=None):
         roger = model.Group.by_name(u'roger')
         model.setup_default_user_roles(david, [russianfan])
         model.setup_default_user_roles(roger, [russianfan])
-        model.add_user_to_role(visitor, model.Role.ADMIN, roger)
 
+        # in new_authz you can't give a visitor permissions to a
+        # group it seems, so this is a bit meaningless
+        model.add_user_to_role(visitor, model.Role.ADMIN, roger)
         model.repo.commit_and_remove()
 
-
     # method used in DGU and all good tests elsewhere
     @classmethod
     def create_users(cls, user_dicts):
@@ -501,9 +546,11 @@ def create_users(cls, user_dicts):
 
     @classmethod
     def _create_user_without_commit(cls, name='', **user_dict):
-        if model.User.by_name(name) or (user_dict.get('open_id') and model.User.by_openid(user_dict.get('openid'))):
-            log.warning('Cannot create user "%s" as it already exists.' % \
-                            (name or user_dict['name']))
+        if model.User.by_name(name) or \
+                (user_dict.get('open_id') and
+                 model.User.by_openid(user_dict.get('openid'))):
+            log.warning('Cannot create user "%s" as it already exists.' %
+                        name or user_dict['name'])
             return
         # User objects are not revisioned so no need to create a revision
         user_ref = name or user_dict['openid']
@@ -826,6 +873,65 @@ def make_some_vocab_tags(cls):
      }
     ]
 
+group_hierarchy_groups = [
+    {'name': 'department-of-health',
+     'title': 'Department of Health',
+     'contact-email': 'contact@doh.gov.uk',
+     'type': 'organization',
+     'is_organization': True
+     },
+    {'name': 'food-standards-agency',
+     'title': 'Food Standards Agency',
+     'contact-email': 'contact@fsa.gov.uk',
+     'parent': 'department-of-health',
+     'type': 'organization',
+     'is_organization': True},
+    {'name': 'national-health-service',
+     'title': 'National Health Service',
+     'contact-email': 'contact@nhs.gov.uk',
+     'parent': 'department-of-health',
+     'type': 'organization',
+     'is_organization': True,
+     'editors': ['nhseditor'],
+     'admins': ['nhsadmin']},
+    {'name': 'nhs-wirral-ccg',
+     'title': 'NHS Wirral CCG',
+     'contact-email': 'contact@wirral.nhs.gov.uk',
+     'parent': 'national-health-service',
+     'type': 'organization',
+     'is_organization': True,
+     'editors': ['wirraleditor'],
+     'admins': ['wirraladmin']},
+    {'name': 'nhs-southwark-ccg',
+     'title': 'NHS Southwark CCG',
+     'contact-email': 'contact@southwark.nhs.gov.uk',
+     'parent': 'national-health-service',
+     'type': 'organization',
+     'is_organization': True},
+    {'name': 'cabinet-office',
+     'title': 'Cabinet Office',
+     'contact-email': 'contact@cabinet-office.gov.uk',
+     'type': 'organization',
+     'is_organization': True},
+    ]
+
+group_hierarchy_datasets = [
+    {'name': 'doh-spend', 'title': 'Department of Health Spend Data',
+     'groups': ['department-of-health']},
+    {'name': 'nhs-spend', 'title': 'NHS Spend Data',
+     'groups': ['national-health-service']},
+    {'name': 'wirral-spend', 'title': 'Wirral Spend Data',
+     'groups': ['nhs-wirral-ccg']},
+    {'name': 'southwark-spend', 'title': 'Southwark Spend Data',
+     'groups': ['nhs-southwark-ccg']},
+    ]
+
+group_hierarchy_users = [{'name': 'nhsadmin', 'password': 'pass'},
+                         {'name': 'nhseditor', 'password': 'pass'},
+                         {'name': 'wirraladmin', 'password': 'pass'},
+                         {'name': 'wirraleditor', 'password': 'pass'},
+                         ]
+
 # Some test terms and translations.
 terms = ('A Novel By Tolstoy',
     'Index of the novel',

ckan/lib/dictization/model_save.py

@@ -342,7 +342,11 @@ def group_member_save(context, group_dict, member_table_name):
     entities = {}
     Member = model.Member
 
-    ModelClass = getattr(model, member_table_name[:-1].capitalize())
+    classname = member_table_name[:-1].capitalize()
+    if classname == 'Organization':
+        # Organizations use the model.Group class
+        classname = 'Group'
+    ModelClass = getattr(model, classname)
 
     for entity_dict in entity_list:
         name_or_id = entity_dict.get('id') or entity_dict.get('name')

ckan/lib/plugins.py

@@ -115,7 +115,7 @@ def register_group_plugins(map):
     """
     Register the various IGroupForm instances.
 
-    This method will setup the mappings between package types and the
+    This method will setup the mappings between group types and the
     registered IGroupForm instances. If it's called more than once an
     exception will be raised.
     """
@@ -160,7 +160,7 @@ def register_group_plugins(map):
 
             if group_type in _group_plugins:
                 raise ValueError, "An existing IGroupForm is "\
-                                  "already associated with the package type "\
+                                  "already associated with the group type "\
                                   "'%s'" % group_type
             _group_plugins[group_type] = plugin
 

ckan/logic/action/create.py

@@ -494,7 +494,6 @@ def _group_or_org_create(context, data_dict, is_org=False):
     model = context['model']
     user = context['user']
     session = context['session']
-    parent = context.get('parent', None)
     data_dict['is_organization'] = is_org
 
     upload = uploader.Upload('group')
@@ -536,14 +535,6 @@ def _group_or_org_create(context, data_dict, is_org=False):
 
     group = model_save.group_dict_save(data, context)
 
-    if parent:
-        parent_group = model.Group.get( parent )
-        if parent_group:
-            member = model.Member(group=parent_group, table_id=group.id, table_name='group')
-            session.add(member)
-            log.debug('Group %s is made child of group %s',
-                      group.name, parent_group.name)
-
     if user:
         admins = [model.User.by_name(user.decode('utf8'))]
     else:

ckan/logic/action/delete.py

@@ -1,5 +1,7 @@
 '''API functions for deleting data from CKAN.'''
 
+from sqlalchemy import or_
+
 import ckan.logic
 import ckan.logic.action
 import ckan.plugins as plugins
@@ -275,6 +277,15 @@ def _group_or_org_delete(context, data_dict, is_org=False):
     rev = model.repo.new_revision()
     rev.author = user
     rev.message = _(u'REST API: Delete %s') % revisioned_details
+
+    # The group's Member objects are deleted
+    # (including hierarchy connections to parent and children groups)
+    for member in model.Session.query(model.Member).\
+            filter(or_(model.Member.table_id == id,
+                       model.Member.group_id == id)).\
+            filter(model.Member.state == 'active').all():
+        member.delete()
+
     group.delete()
 
     if is_org:

ckan/logic/action/update.py

@@ -417,7 +417,6 @@ def _group_or_org_update(context, data_dict, is_org=False):
     user = context['user']
     session = context['session']
     id = _get_or_bust(data_dict, 'id')
-    parent = context.get('parent', None)
 
     group = model.Group.get(id)
     context["group"] = group
@@ -477,23 +476,6 @@ def _group_or_org_update(context, data_dict, is_org=False):
         context['prevent_packages_update'] = True
     group = model_save.group_dict_save(data, context)
 
-    if parent:
-        parent_group = model.Group.get( parent )
-        if parent_group and not parent_group in group.get_groups(group.type):
-            # Delete all of this groups memberships
-            current = session.query(model.Member).\
-               filter(model.Member.table_id == group.id).\
-               filter(model.Member.table_name == "group").all()
-            if current:
-                log.debug('Parents of group %s deleted: %r', group.name,
-                          [membership.group.name for membership in current])
-            for c in current:
-                session.delete(c)
-            member = model.Member(group=parent_group, table_id=group.id, table_name='group')
-            session.add(member)
-            log.debug('Group %s is made child of group %s',
-                      group.name, parent_group.name)
-
     if is_org:
         plugin_type = plugins.IOrganizationController
     else: