Add export archive migration for Group type strings

The `Group` entity class is now subclassable, for which the type string
of existing entries in the database had to be changed through a
migration. Here we add the corresponding migration for existing export
archives. The only required change is to map the type string of groups.

To test this migration we use the existing export archives in the module
`tests/fixtures/export/migrate`. They did not contain any group entities
so `export_v0.1_simple.aiida` was updated first and had four groups
added, one for each of the migrated type strings. This initial archive
was then migrated to each subsequent version, step by step, using the
command `verdi export migrate --version`.

Also fixed and migrated `tests/fixtures/graphs/graph1.aiida` which was
corrupt and could not be migrated automatically, because it contained a
process node with an active process state.

aiida/tools/importexport/common/config.py

@@ -14,7 +14,7 @@
 __all__ = ('EXPORT_VERSION',)
 
 # Current export version
-EXPORT_VERSION = '0.8'
+EXPORT_VERSION = '0.9'
 
 DUPL_SUFFIX = ' (Imported #{})'
 

aiida/tools/importexport/migration/__init__.py

@@ -20,6 +20,7 @@
 from .v05_to_v06 import migrate_v5_to_v6
 from .v06_to_v07 import migrate_v6_to_v7
 from .v07_to_v08 import migrate_v7_to_v8
+from .v08_to_v09 import migrate_v8_to_v9
 
 __all__ = ('migrate_recursively', 'verify_metadata_version')
 
@@ -31,6 +32,7 @@
     '0.5': migrate_v5_to_v6,
     '0.6': migrate_v6_to_v7,
     '0.7': migrate_v7_to_v8,
+    '0.8': migrate_v8_to_v9,
 }
 
 

aiida/tools/importexport/migration/v08_to_v09.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""Migration from v0.8 to v0.9, used by `verdi export migrate` command.
+
+The migration steps are named similarly to the database migrations for Django and SQLAlchemy.
+In the description of each migration, a revision number is given, which refers to the Django migrations.
+The individual Django database migrations may be found at:
+
+    `aiida.backends.djsite.db.migrations.00XX_<migration-name>.py`
+
+Where XX are the numbers in the migrations' documentation: REV. 1.0.XX
+And migration-name is the name of the particular migration.
+The individual SQLAlchemy database migrations may be found at:
+
+    `aiida.backends.sqlalchemy.migrations.versions.<id>_<migration-name>.py`
+
+Where id is a SQLA id and migration-name is the name of the particular migration.
+"""
+# pylint: disable=invalid-name
+
+from aiida.tools.importexport.migration.utils import verify_metadata_version, update_metadata
+
+
+def migration_dbgroup_type_string(data):
+    """Apply migration 0044 - REV. 1.0.44
+
+    Rename the `type_string` columns of all `Group` instances.
+    """
+    mapping = {
+        'user': 'core',
+        'data.upf': 'core.upf',
+        'auto.import': 'core.import',
+        'auto.run': 'core.auto',
+    }
+
+    for attributes in data.get('export_data', {}).get('Group', {}).values():
+        for old, new in mapping.items():
+            if attributes['type_string'] == old:
+                attributes['type_string'] = new
+
+
+def migrate_v8_to_v9(metadata, data, *args):  # pylint: disable=unused-argument
+    """Migration of export files from v0.8 to v0.9."""
+    old_version = '0.8'
+    new_version = '0.9'
+
+    verify_metadata_version(metadata, old_version)
+    update_metadata(metadata, new_version)
+
+    # Apply migrations
+    migration_dbgroup_type_string(data)

docs/requirements_for_rtd.txt

@@ -1,5 +1,5 @@
 PyCifRW~=4.4
-aiida-export-migration-tests==0.8.0
+aiida-export-migration-tests==0.9.0
 aldjemy~=0.9.1
 alembic~=1.2
 ase~=3.18

requirements/requirements-py-3.5.txt

@@ -1,4 +1,4 @@
-aiida-export-migration-tests==0.8.0
+aiida-export-migration-tests==0.9.0
 alabaster==0.7.12
 aldjemy==0.9.1
 alembic==1.4.1

requirements/requirements-py-3.6.txt

@@ -1,4 +1,4 @@
-aiida-export-migration-tests==0.8.0
+aiida-export-migration-tests==0.9.0
 alabaster==0.7.12
 aldjemy==0.9.1
 alembic==1.4.1

requirements/requirements-py-3.7.txt

@@ -1,4 +1,4 @@
-aiida-export-migration-tests==0.8.0
+aiida-export-migration-tests==0.9.0
 alabaster==0.7.12
 aldjemy==0.9.1
 alembic==1.4.1

requirements/requirements-py-3.8.txt

@@ -1,4 +1,4 @@
-aiida-export-migration-tests==0.8.0
+aiida-export-migration-tests==0.9.0
 alabaster==0.7.12
 aldjemy==0.9.1
 alembic==1.4.1

setup.json

@@ -88,7 +88,7 @@
       "notebook<6"
     ],
     "testing": [
-      "aiida-export-migration-tests==0.8.0",
+      "aiida-export-migration-tests==0.9.0",
       "pg8000~=1.13",
       "pgtest~=1.3,>=1.3.1",
       "pytest~=5.3",

tests/tools/importexport/migration/test_v08_to_v09.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""Test export file migration from export version 0.8 to 0.9"""
+from aiida.tools.importexport.migration.v08_to_v09 import migrate_v8_to_v9, migration_dbgroup_type_string
+
+from . import ArchiveMigrationTest
+
+
+class TestMigrate(ArchiveMigrationTest):
+    """Tests specific for this archive migration."""
+
+    def test_migrate_external(self):
+        """Test the migration on the test archive provided by the external test package."""
+        _, data = self.migrate('export_v0.8_manual.aiida', '0.8', '0.9', migrate_v8_to_v9)
+
+        for attributes in data.get('export_data', {}).get('Group', {}).values():
+            if attributes['type_string'] not in ['core', 'core.upf', 'core.import', 'core.auto']:
+                raise AssertionError('encountered illegal type string `{}`'.format(attributes['type_string']))
+
+    def test_migration_dbgroup_type_string(self):
+        """Test the `migration_dbgroup_type_string` function directly."""
+
+        data = {
+            'export_data': {
+                'Group': {
+                    '50': {
+                        'type_string': 'user',
+                    },
+                    '51': {
+                        'type_string': 'data.upf',
+                    },
+                    '52': {
+                        'type_string': 'auto.import',
+                    },
+                    '53': {
+                        'type_string': 'auto.run',
+                    }
+                }
+            }
+        }
+
+        migration_dbgroup_type_string(data)
+
+        self.assertEqual(
+            data, {
+                'export_data': {
+                    'Group': {
+                        '50': {
+                            'type_string': 'core',
+                        },
+                        '51': {
+                            'type_string': 'core.upf',
+                        },
+                        '52': {
+                            'type_string': 'core.import',
+                        },
+                        '53': {
+                            'type_string': 'core.auto',
+                        }
+                    }
+                }
+            }
+        )