Francedata fixes

.travis.yml

@@ -10,7 +10,6 @@ install:
 - pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov django-responsediff
 - pip install -e .[api]
 script:
-- pep8 representatives/ --exclude migrations --ignore E128
 - flake8 representatives/ --exclude migrations --ignore E128
 - django-admin migrate
 - cat representatives/contrib/francedata/tests/representatives_input.json | francedata_import_representatives

representatives/contrib/francedata/import_representatives.py

@@ -9,6 +9,7 @@
 import django
 from django.apps import apps
 from django.db import transaction
+from django.db import DatabaseError
 from django.utils import timezone
 
 from representatives.models import (Country, Mandate, Email, Address, WebSite,
@@ -43,21 +44,6 @@ def _parse_date(date):
     return datetime.strptime(date, "%Y-%m-%d").date()
 
 
-def _create_mandate(representative, group, constituency, role='',
-                    begin_date=None, end_date=None):
-    mandate, _ = Mandate.objects.get_or_create(
-        representative=representative,
-        group=group,
-        constituency=constituency,
-        role=role,
-        begin_date=begin_date,
-        end_date=end_date
-    )
-
-    if _:
-        logger.debug('Created mandate %s', mandate.pk)
-
-
 def _get_path(dict_, path):
     '''
     Get value at specific path in dictionary. Path is specified by slash-
@@ -80,12 +66,25 @@ def touch_model(self, model, **data):
         it saves the given model if it exists, updating its
         updated field
         '''
-
-        instance, created = model.objects.get_or_create(**data)
+        model_name = model._meta.verbose_name
+        data_string = repr(data).decode('utf8')
+        logger.debug('Saving %s with %s' % (model_name, data_string))
+        try:
+            with transaction.atomic():
+                instance, created = model.objects.get_or_create(**data)
+        except DatabaseError as e:
+            logger.error('Saving %s failed: %s' % (model_name, e))
+            return False, False
 
         if not created:
             if instance.updated < self.import_start_datetime:
-                instance.save()     # Updates updated field
+                logger.debug('Updating %s with %s' % (model_name, data_string))
+                try:
+                    with transaction.atomic():
+                        instance.save()
+                except DatabaseError as e:
+                    logger.error('Saving %s failed: %s' % (model_name, e))
+                    return False, False
 
         return (instance, created)
 
@@ -180,15 +179,18 @@ def add_mandates(self, representative, rep_json):
 
         # Mandate in country group for party constituency
         if rep_json.get('parti_ratt_financier'):
-            constituency, _ = Constituency.objects.get_or_create(
-                name=rep_json.get('parti_ratt_financier'), country=self.france)
+            constituency, _ = self.touch_model(
+                model=Constituency,
+                name=rep_json.get('parti_ratt_financier'),
+                country=self.france
+            )
 
             group, _ = self.touch_model(model=Group,
                                         abbreviation=self.france.code,
                                         kind='country',
                                         name=self.france.name)
 
-            _create_mandate(representative, group, constituency, 'membre')
+            self._create_mandate(representative, group, constituency, 'membre')
 
         # Configurable mandates
         for mdef in self.variant['mandates']:
@@ -212,6 +214,9 @@ def add_mandates(self, representative, rep_json):
                                             chamber=chamber,
                                             name=name)
 
+                if group is False:
+                    continue
+
                 role = _get_mdef_item(mdef, 'role', elem, 'membre')
                 start = _get_mdef_item(mdef, 'start', elem, None)
                 if start is not None:
@@ -220,8 +225,8 @@ def add_mandates(self, representative, rep_json):
                 if end is not None:
                     end = _parse_date(end)
 
-                _create_mandate(representative, group, self.ch_constituency,
-                                role, start, end)
+                self._create_mandate(representative, group,
+                        self.ch_constituency, role, start, end)
 
                 logger.debug(
                     '%s => %s: %s of "%s" (%s) %s-%s' % (rep_json['slug'],
@@ -300,6 +305,18 @@ def add_contacts(self, representative, rep_json):
                                  kind='', number=item['tel']
                                  )
 
+    def _create_mandate(self, representative, group, constituency, role='',
+                        begin_date=None, end_date=None):
+        mandate, _ = self.touch_model(
+            model=Mandate,
+            representative=representative,
+            group=group,
+            constituency=constituency,
+            role=role,
+            begin_date=begin_date,
+            end_date=end_date
+        )
+
 
 def main(stream=None):
     if not apps.ready:

representatives/migrations/0017_increment_mandate_role_size.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('representatives', '0016_chamber_migrate_data'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mandate',
+            name='role',
+            field=models.CharField(default=b'', help_text=b'Eg.: president of a political group', max_length=255, blank=True),
+        ),
+    ]

representatives/models.py

@@ -175,6 +175,11 @@ class Chamber(HashableModel):
 
     hashable_fields = ['name', 'country', 'abbreviation']
 
+    class Meta:
+        unique_together = [
+            ['name', 'country', 'abbreviation']
+        ]
+
     def __unicode__(self):
         return u'{} [{}]'.format(self.name, self.abbreviation)
 
@@ -200,6 +205,9 @@ def __unicode__(self):
 
     class Meta:
         ordering = ('name',)
+        unique_together = [
+            ['name', 'abbreviation', 'kind', 'chamber']
+        ]
 
     def get_absolute_url(self):
         if self.chamber:
@@ -219,7 +227,7 @@ class Constituency(HashableModel, TimeStampedModel):
     country = models.ForeignKey('Country', null=True, blank=True,
         related_name='constituencies')
 
-    hashable_fields = ['name']
+    hashable_fields = ['name', 'country']
 
     @cached_property
     def active(self):
@@ -248,7 +256,7 @@ class Mandate(HashableModel, TimeStampedModel):
         Constituency, null=True, related_name='mandates')
     representative = models.ForeignKey(Representative, related_name='mandates')
     role = models.CharField(
-        max_length=25,
+        max_length=255,
         blank=True,
         default='',
         help_text="Eg.: president of a political group"
@@ -276,3 +284,9 @@ def __unicode__(self):
 
     class Meta:
         ordering = ('-end_date',)
+        unique_together = [
+            [
+                'group', 'constituency', 'role', 'begin_date',
+                'end_date', 'representative'
+            ]
+        ]