Merge pull request #274 from c3g/master-on-qc-38

merge of master on qc for release 3.8

backend/README.md

@@ -131,7 +131,7 @@ coverage run ./manage.py test -v 2
 
 ## Database diagram
 
-[Database Schema Diagram](https://dbdiagram.io/d/62012be785022f4ee54f72f3)
+[Database Schema Diagram](https://dbdiagram.io/d/622116d754f9ad109a563cbc)
 
 
 ## pg_fzy

backend/VERSION

@@ -1 +1 @@
-3.7.1
+3.8.0

backend/fms_core/filters.py

@@ -10,6 +10,7 @@
     _individual_filterset_fields,
     _sample_minimal_filterset_fields,
     _index_filterset_fields,
+    _library_filterset_fields,
 )
 
 from .viewsets._utils import _prefix_keys
@@ -40,21 +41,50 @@ class SampleFilter(GenericFilter):
     container__barcode = django_filters.CharFilter(field_name="container__barcode", method="batch_filter")
     qPCR_status__in = django_filters.CharFilter(method="process_measurement_properties_filter")
     projects__name = django_filters.CharFilter(method="batch_filter")
+    qc_flag__in = django_filters.CharFilter(method="qc_flag_filter")
 
     def process_measurement_properties_filter(self, queryset, name, value):
         property_values = PropertyValue.objects.filter(property_type__name='qPCR Status')
         condition = Q()
         for status in value.split(','):
             condition |= Q(value__icontains=status)
-            process_measurements_ids = property_values.filter(condition).values('object_id')
-            return queryset.filter(process_measurement__in=process_measurements_ids)
-        else:
-            return queryset
+        process_measurements_ids = property_values.filter(condition).values('object_id')
+        return queryset.filter(process_measurement__in=process_measurements_ids)
+
+    def qc_flag_filter(self, queryset, name, values):
+        condition = Q()
+        for value in values.split(','):
+            if value == "None":
+                bool_value = None
+            else:
+                bool_value = (value == 'true')
+            condition |= Q(qc_flag=bool_value)
+        return queryset.filter(condition)
 
     class Meta:
         model = Sample
         fields = _sample_filterset_fields
 
+class LibraryFilter(GenericFilter):
+    name = django_filters.CharFilter(field_name="name", method="batch_filter")
+    container__barcode = django_filters.CharFilter(field_name="container__barcode", method="batch_filter")
+    projects__name = django_filters.CharFilter(method="batch_filter")
+    qc_flag__in = django_filters.CharFilter(method="qc_flag_filter")
+
+    def qc_flag_filter(self, queryset, name, values):
+        condition = Q()
+        for value in values.split(','):
+            if value == "None":
+                bool_value = None
+            else:
+                bool_value = (value == 'true')
+            condition |= Q(qc_flag=bool_value)
+        return queryset.filter(condition)
+
+    class Meta:
+        model = Sample
+        fields = _library_filterset_fields
+
 class IndividualFilter(GenericFilter):
     name = django_filters.CharFilter(field_name="name", method="batch_filter")
 

backend/fms_core/migrations/0033_v3_8_0.py

@@ -0,0 +1,155 @@
+from django.conf import settings
+import django.core.validators
+from django.contrib.auth.models import User
+from django.db import migrations, models
+import django.db.models.deletion
+import re
+import reversion
+
+ADMIN_USERNAME = 'biobankadmin'
+
+def create_library_preparation_objects(apps, schema_editor):
+    Protocol = apps.get_model("fms_core", "Protocol")
+    PropertyType = apps.get_model("fms_core", "PropertyType")
+    ContentType = apps.get_model('contenttypes', 'ContentType')
+
+    with reversion.create_revision(manage_manually=True):
+        admin_user = User.objects.get(username=ADMIN_USERNAME)
+        admin_user_id = admin_user.id
+
+        reversion.set_comment("Create objects related to the Library preparation protocol")
+        reversion.set_user(admin_user)
+
+        # Create PropertyType and Protocols
+        PROPERTY_TYPES_BY_PROTOCOL = {
+            "Library Preparation": [
+                                  ("Library Technician Name", "str"),
+                                  ("Shearing Technician Name", "str"),
+                                  ("Shearing Method", "str"),
+                                  ("Shearing Size (bp)", "str"),
+                                  ("Library Kit Used", "str"),
+                                  ("Library Kit Lot", "str"),
+                                  ("Thermocycler Used", "str"),
+                                  ("PCR Cycles", "str"),
+                                  ("PCR Enzyme Used", "str"),
+                                  ("PCR Enzyme Lot", "str"),
+                                  ("EZ-96 DNA Methylation-Gold MagPrep Lot", "str")
+                                 ]
+        }
+        protocol_content_type = ContentType.objects.get_for_model(Protocol)
+
+        for protocol_name in PROPERTY_TYPES_BY_PROTOCOL.keys():
+            protocol = Protocol.objects.create(name=protocol_name, created_by_id=admin_user_id, updated_by_id=admin_user_id)
+            reversion.add_to_revision(protocol)
+
+            for (property, value_type) in PROPERTY_TYPES_BY_PROTOCOL[protocol_name]:
+                if any([property == 'Library Technician Name', property == 'Library Kit Used', property == 'Library Kit Lot']):
+                    is_optional = False
+                else:
+                    is_optional = True
+                pt = PropertyType.objects.create(name=property,
+                                                 object_id=protocol.id,
+                                                 content_type=protocol_content_type,
+                                                 value_type=value_type,
+                                                 is_optional=is_optional,
+                                                 created_by_id=admin_user_id, updated_by_id=admin_user_id)
+                reversion.add_to_revision(pt)
+
+def create_initial_library_types(apps, schema_editor):
+    LibraryType = apps.get_model("fms_core", "LibraryType")
+
+    LIBRARY_TYPES = ["PCR-free",
+                     "PCR-enriched",
+                     "RNASeq",
+                     "WGBS",
+                     "16S",
+                     "18S",
+                     "miRNA",]
+
+    with reversion.create_revision(manage_manually=True):
+        admin_user = User.objects.get(username=ADMIN_USERNAME)
+        admin_user_id = admin_user.id
+
+        reversion.set_comment("Create objects related to the Sample Selection using qPCR protocol")
+        reversion.set_user(admin_user)
+
+        for library_type_name in LIBRARY_TYPES:
+            library_type = LibraryType.objects.create(name=library_type_name, created_by_id=admin_user_id, updated_by_id=admin_user_id)
+            reversion.add_to_revision(library_type)
+
+def add_tissue_sample_kind(apps, schema_editor):
+    SampleKind = apps.get_model("fms_core", "SampleKind")
+    with reversion.create_revision(manage_manually=True):
+        admin_id = User.objects.get(username="biobankadmin").id
+        tissue_sample_kind = SampleKind.objects.create(name='TISSUE', created_by_id=admin_id, updated_by_id=admin_id)
+        reversion.add_to_revision(tissue_sample_kind)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('fms_core', '0032_v3_7_0'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            create_library_preparation_objects,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.RemoveField(
+            model_name='derivedsample',
+            name='index',
+        ),
+        migrations.CreateModel(
+            name='LibraryType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Date the instance was created.')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='Date the instance was modified.')),
+                ('deleted', models.BooleanField(default=False, help_text='Whether this instance has been deleted.')),
+                ('name', models.CharField(help_text='The name of the library type.', max_length=200, unique=True, validators=[django.core.validators.RegexValidator(re.compile('^[a-zA-Z0-9.\\-_]{1,200}$'))])),
+                ('created_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_librarytype_creation', to=settings.AUTH_USER_MODEL)),
+                ('updated_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_librarytype_modification', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Library',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Date the instance was created.')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='Date the instance was modified.')),
+                ('deleted', models.BooleanField(default=False, help_text='Whether this instance has been deleted.')),
+                ('library_size', models.DecimalField(blank=True, decimal_places=0, help_text='Average size of the nucleic acid strands in base pairs.', max_digits=20, null=True)),
+                ('created_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_library_creation', to=settings.AUTH_USER_MODEL)),
+                ('index', models.ForeignKey(help_text='The index associated to this library.', on_delete=django.db.models.deletion.PROTECT, related_name='libraries', to='fms_core.index')),
+                ('library_type', models.ForeignKey(help_text='Library type describing the library.', on_delete=django.db.models.deletion.PROTECT, related_name='libraries', to='fms_core.librarytype')),
+                ('platform', models.ForeignKey(help_text='The platform for which the library has been prepared.', on_delete=django.db.models.deletion.PROTECT, related_name='libraries', to='fms_core.platform')),
+                ('strandedness', models.CharField(choices=[('Double stranded', 'Double stranded'), ('Single stranded', 'Single stranded')], help_text='Number of Library NA strands.', max_length=20)),
+                ('updated_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_library_modification', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='derivedsample',
+            name='library',
+            field=models.OneToOneField(blank=True, help_text='Library associated to this Derived Sample.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='derived_sample', to='fms_core.library'),
+        ),
+        migrations.RunPython(
+            create_initial_library_types,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.RunPython(
+            add_tissue_sample_kind,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.AlterField(
+            model_name='derivedsample',
+            name='tissue_source',
+            field=models.CharField(blank=True, choices=[('BAL', 'BAL'), ('Biopsy', 'Biopsy'), ('Blood', 'Blood'), ('Buffy coat', 'Buffy coat'), ('Cells', 'Cells'), ('Expectoration', 'Expectoration'), ('Gargle', 'Gargle'), ('Plasma', 'Plasma'), ('Saliva', 'Saliva'), ('Swab', 'Swab'), ('Tail', 'Tail'), ('Tissue', 'Tissue'), ('Tumor', 'Tumor')], help_text='Can only be specified if the biospecimen type is DNA or RNA.', max_length=200),
+        ),
+    ]

backend/fms_core/migrations/0034_v3_8_0.py

@@ -0,0 +1,126 @@
+from django.conf import settings
+import django.core.validators
+from django.contrib.auth.models import User
+from django.db import migrations, models
+from django.db.models import Q
+import django.db.models.deletion
+import re
+import reversion
+
+ADMIN_USERNAME = 'biobankadmin'
+
+def initialize_taxons(apps, schema_editor):
+    Taxon = apps.get_model("fms_core", "Taxon")
+    Individual = apps.get_model("fms_core", "Individual")
+
+    # List of existing Taxons in Freezeman
+    TAXON_HOMO_SAPIENS = "Homo sapiens"
+    TAXON_MUS_MUSCULUS = "Mus musculus"
+    TAXON_SARS_COV_2 = "Sars-Cov-2"
+    TAXON_IXODES_SCAPULARIS = "Ixodes scapularis"
+
+    taxon_dict = { TAXON_HOMO_SAPIENS: 9606,
+                   TAXON_MUS_MUSCULUS: 10090,
+                   TAXON_SARS_COV_2: 2697049,
+                   TAXON_IXODES_SCAPULARIS: 6945, }
+
+    taxon_map = {}
+
+    with reversion.create_revision(manage_manually=True):
+        admin_user = User.objects.get(username=ADMIN_USERNAME)
+        admin_user_id = admin_user.id
+
+        reversion.set_comment("Create taxons to support taxons currently used in Freezeman.")
+        reversion.set_user(admin_user)
+
+        for name, ncbi_id in taxon_dict.items():
+            taxon = Taxon.objects.create(name=name, ncbi_id=ncbi_id, created_by_id=admin_user_id, updated_by_id=admin_user_id)
+            reversion.add_to_revision(taxon)
+            taxon_map[name] = taxon.id
+
+        for individual in Individual.objects.all():
+            individual.taxon_new = taxon_map[individual.taxon]
+            individual.save()
+            reversion.add_to_revision(individual)
+
+def transfer_qc_flags(apps, schema_editor):
+    Sample = apps.get_model("fms_core", "Sample")
+    with reversion.create_revision(manage_manually=True):
+        samples_with_qc_flags = Sample.objects.filter(Q(derived_samples__quality_flag__isnull=False) | Q(derived_samples__quantity_flag__isnull=False))
+        for sample in samples_with_qc_flags:
+            sample.quality_flag = sample.derived_samples.first().quality_flag
+            sample.quantity_flag = sample.derived_samples.first().quantity_flag
+            sample.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('fms_core', '0033_v3_8_0'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sample',
+            name='quality_flag',
+            field=models.BooleanField(blank=True, choices=[(True, 'Passed'), (False, 'Failed')],
+                                      help_text='Quality flag of the sample.', max_length=20, null=True),
+        ),
+        migrations.AddField(
+            model_name='sample',
+            name='quantity_flag',
+            field=models.BooleanField(blank=True, choices=[(True, 'Passed'), (False, 'Failed')],
+                                      help_text='Quantity flag of the sample.', max_length=20, null=True),
+        ),
+        migrations.RunPython(
+            transfer_qc_flags,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.RemoveField(
+            model_name='derivedsample',
+            name='quality_flag',
+        ),
+        migrations.RemoveField(
+            model_name='derivedsample',
+            name='quantity_flag',
+        ),
+        migrations.CreateModel(
+            name='Taxon',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Date the instance was created.')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='Date the instance was modified.')),
+                ('deleted', models.BooleanField(default=False, help_text='Whether this instance has been deleted.')),
+                ('name', models.CharField(help_text='Taxon scientific name.', max_length=200, unique=True)),
+                ('ncbi_id', models.PositiveBigIntegerField(help_text='Numerical identifier used by the NCBI taxonomy catalog.', unique=True)),
+                ('created_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_taxon_creation', to=settings.AUTH_USER_MODEL)),
+                ('updated_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_taxon_modification', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='individual',
+            name='taxon_new',
+            field=models.IntegerField(blank=True, null=True, help_text='Taxonomic entry associated to the individual.'),
+        ),
+        migrations.RunPython(
+            initialize_taxons,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.RemoveField(
+            model_name='individual',
+            name='taxon',
+        ),
+        migrations.RenameField(
+            model_name='individual',
+            old_name='taxon_new',
+            new_name='taxon',
+        ),
+        migrations.AlterField(
+            model_name='individual',
+            name='taxon',
+            field=models.ForeignKey(help_text='Taxonomic entry associated to the individual.', on_delete=django.db.models.deletion.PROTECT, to='fms_core.taxon'),
+        ),
+    ]

backend/fms_core/migrations/0035_v3_8_0.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.1 on 2022-04-11 13:21
+
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import re
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('fms_core', '0034_v3_8_0'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SampleMetadata',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Date the instance was created.')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='Date the instance was modified.')),
+                ('deleted', models.BooleanField(default=False, help_text='Whether this instance has been deleted.')),
+                ('name', models.CharField(help_text='The name of the metadata.', max_length=200, validators=[django.core.validators.RegexValidator(re.compile('^[a-zA-Z0-9.\\-_]{1,200}$'))])),
+                ('value', models.CharField(help_text='The value of the metadata.', max_length=2000)),
+                ('created_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_samplemetadata_creation', to=settings.AUTH_USER_MODEL)),
+                ('biosample', models.ForeignKey(help_text='Biosample associated to this metadata.', on_delete=django.db.models.deletion.PROTECT, related_name='metadata', to='fms_core.biosample')),
+                ('updated_by', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='fms_core_samplemetadata_modification', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='samplemetadata',
+            constraint=models.UniqueConstraint(fields=('name', 'biosample'), name='unique_metadata_name_by_biosample'),
+        ),
+    ]