[r] Rename SourceName to SourceSpec (#2843, PR #3047)

deployments/sandbox/environment.py

@@ -96,8 +96,6 @@ def env() -> Mapping[str, Optional[str]]:
             }
         ),
 
-        'AZUL_DSS_QUERY_PREFIX': '42',
-
         'AZUL_DEBUG': '1',
 
         # When using this file as a template for a personal deployment, change

scripts/can_bundle.py

@@ -69,7 +69,7 @@ def fetch_bundle(source: str, bundle_uuid: str, bundle_version: str) -> Bundle:
         plugin = plugin_for(catalog)
         sources = set(map(str, plugin.sources))
         if source in sources:
-            source = plugin.resolve_source(name=source)
+            source = plugin.resolve_source(spec=source)
             fqid = SourcedBundleFQID(source=source,
                                      uuid=bundle_uuid,
                                      version=bundle_version)

scripts/recan_bundle_tdr.py

@@ -35,7 +35,7 @@
 )
 from azul.indexer import (
     Bundle,
-    SimpleSourceName,
+    SimpleSourceSpec,
     SourcedBundleFQID,
 )
 from azul.indexer.document import (
@@ -60,7 +60,7 @@
     TDRSourceRef,
 )
 from azul.terra import (
-    TDRSourceName,
+    TDRSourceSpec,
 )
 from azul.types import (
     JSON,
@@ -237,7 +237,7 @@ def __init__(self, bundle: TDRBundle, file_name: str):
         assert self.concrete_type.endswith('_file')
         self.file_manifest_entry = one(e for e in bundle.manifest
                                        if e['name'] == self.metadata['file_core']['file_name'])
-        assert bundle.fqid.source.name.is_snapshot
+        assert bundle.fqid.source.spec.is_snapshot
         assert self.file_manifest_entry['drs_path'] is not None
 
     def to_json_row(self) -> JSON:
@@ -387,7 +387,7 @@ def main(argv):
         metadata = json.load(f)
 
     dss_source = DSSSourceRef(id='',
-                              name=SimpleSourceName(prefix='',
+                              spec=SimpleSourceSpec(prefix='',
                                                     name=config.dss_endpoint))
     dss_bundle = DSSBundle(fqid=SourcedBundleFQID(source=dss_source,
                                                   uuid=args.bundle_uuid,
@@ -396,7 +396,7 @@ def main(argv):
                            metadata_files=metadata)
 
     tdr_source = TDRSourceRef(id=args.source_id,
-                              name=TDRSourceName(project='test_project',
+                              spec=TDRSourceSpec(project='test_project',
                                                  name='test_name',
                                                  is_snapshot=True))
     tdr_bundle = dss_bundle_to_tdr(dss_bundle, tdr_source)

scripts/register_sam.py

@@ -12,7 +12,7 @@
 )
 from azul.terra import (
     TDRClient,
-    TDRSourceName,
+    TDRSourceSpec,
 )
 
 log = logging.getLogger(__name__)
@@ -29,7 +29,7 @@ def main():
         if catalog.plugins['repository'] == 'tdr'
     )
     for source in set(chain(*map(config.tdr_sources, tdr_catalogs))):
-        source = TDRSourceName.parse(source)
+        source = TDRSourceSpec.parse(source)
         api_project = tdr.lookup_source_project(source)
         require(api_project == source.project,
                 'Actual Google project of TDR source differs from configured '

src/azul/azulclient.py

@@ -103,7 +103,7 @@ def synthesize_notification(self,
         return {
             'source': {
                 'id': bundle_fqid.source.id,
-                'name': str(bundle_fqid.source.name),
+                'spec': str(bundle_fqid.source.spec),
             },
             'query': self.query(catalog, prefix),
             'subscription_id': 'cafebabe-feed-4bad-dead-beaf8badf00d',
@@ -212,7 +212,7 @@ def list_bundles(self,
                      ) -> List[SourcedBundleFQID]:
         validate_uuid_prefix(prefix)
         plugin = self.repository_plugin(catalog)
-        source = plugin.resolve_source(name=source)
+        source = plugin.resolve_source(spec=source)
         return plugin.list_bundles(source, prefix)
 
     @property
@@ -292,8 +292,8 @@ def filter_obsolete_bundle_versions(cls,
         each bundle UUID.
         >>> AzulClient.filter_obsolete_bundle_versions([])
         []
-        >>> from azul.indexer import SimpleSourceName, SourceRef
-        >>> s = SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n'))
+        >>> from azul.indexer import SimpleSourceSpec, SourceRef
+        >>> s = SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n'))
         >>> def b(u, v):
         ...     return SourcedBundleFQID(source=s, uuid=u, version=v)
         >>> AzulClient.filter_obsolete_bundle_versions([
@@ -303,32 +303,32 @@ def filter_obsolete_bundle_versions(cls,
         ... ]) # doctest: +NORMALIZE_WHITESPACE
         [SourcedBundleFQID(uuid='c',
                            version='0',
-                           source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n'))),
+                           source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n'))),
         SourcedBundleFQID(uuid='b',
                           version='3',
-                          source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n'))),
+                          source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n'))),
         SourcedBundleFQID(uuid='a',
                           version='1',
-                          source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n')))]
+                          source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n')))]
         >>> AzulClient.filter_obsolete_bundle_versions([
         ...     b('C', '0'), b('a', '1'), b('a', '0'),
         ...     b('a', '2'), b('b', '1'), b('c', '2')
         ... ]) # doctest: +NORMALIZE_WHITESPACE
         [SourcedBundleFQID(uuid='c',
                            version='2',
-                           source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n'))),
+                           source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n'))),
         SourcedBundleFQID(uuid='b',
                            version='1',
-                           source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n'))),
+                           source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n'))),
         SourcedBundleFQID(uuid='a',
                            version='2',
-                           source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n')))]
+                           source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n')))]
         >>> AzulClient.filter_obsolete_bundle_versions([
         ...     b('a', '0'), b('A', '1')
         ... ]) # doctest: +NORMALIZE_WHITESPACE
         [SourcedBundleFQID(uuid='A',
                            version='1',
-                           source=SourceRef(id='i', name=SimpleSourceName(prefix='42', name='n')))]
+                           source=SourceRef(id='i', spec=SimpleSourceSpec(prefix='42', name='n')))]
         """
 
         # Sort lexicographically by source and FQID. I've observed the DSS

src/azul/indexer/__init__.py

@@ -21,6 +21,7 @@
 import attr
 
 from azul import (
+    reject,
     require,
 )
 from azul.types import (
@@ -31,6 +32,10 @@
 
 # FIXME: Remove hacky import of SupportsLessThan
 #        https://github.com/DataBiosphere/azul/issues/2783
+from azul.uuids import (
+    validate_uuid_prefix,
+)
+
 if TYPE_CHECKING:
     from _typeshed import (
         SupportsLessThan,
@@ -50,25 +55,27 @@ class BundleFQID(SupportsLessThan):
     version: BundleVersion
 
 
-SOURCE_NAME = TypeVar('SOURCE_NAME', bound='SourceName')
+SOURCE_SPEC = TypeVar('SOURCE_SPEC', bound='SourceSpec')
 
 
-# FIXME: Rename to SourceSpec/SOURCE_SPEC, and all .name to .spec
-#        https://github.com/DataBiosphere/azul/issues/2843
 @attr.s(frozen=True, auto_attribs=True, kw_only=True)
-class SourceName(ABC, Generic[SOURCE_NAME]):
+class SourceSpec(ABC, Generic[SOURCE_SPEC]):
     """
     The name of a repository source containing bundles to index. A repository
     has at least one source. Repository plugins whose repository source names
     are structured might want to implement this abstract class. Plugins that
-    have simple unstructured names may want to use :class:`StringSourceName`.
+    have simple unstructured names may want to use :class:`SimpleSourceSpec`.
     """
 
-    prefix: Optional[str] = ''
+    prefix: str = ''
+
+    def __attrs_post_init__(self):
+        validate_uuid_prefix(self.prefix)
+        assert ':' not in self.prefix, self.prefix
 
     @classmethod
     @abstractmethod
-    def parse(cls, name: str) -> SOURCE_NAME:
+    def parse(cls, spec: str) -> SOURCE_SPEC:
         raise NotImplementedError
 
     @abstractmethod
@@ -77,25 +84,54 @@ def __str__(self) -> str:
 
 
 @attr.s(frozen=True, auto_attribs=True, kw_only=True)
-class SimpleSourceName(SourceName['SimpleSourceName']):
+class SimpleSourceSpec(SourceSpec['SimpleSourceSpec']):
     """
     Default implementation for unstructured source names.
     """
     name: str
 
     @classmethod
-    def parse(cls, name: str) -> 'SimpleSourceName':
-        return cls(name=name)
+    def parse(cls, spec: str) -> 'SimpleSourceSpec':
+        """
+        >>> SimpleSourceSpec.parse('https://foo.edu:12')
+        SimpleSourceSpec(prefix='12', name='https://foo.edu')
+
+        >>> SimpleSourceSpec.parse('foo')
+        Traceback (most recent call last):
+        ...
+        azul.RequirementError: Source specifications must end in a colon followed by an optional UUID prefix
+
+        >>> SimpleSourceSpec.parse('foo:8F53')
+        Traceback (most recent call last):
+        ...
+        azul.uuids.InvalidUUIDPrefixError: '8F53' is not a valid UUID prefix.
+
+        >>> SimpleSourceSpec.parse('https://foo.edu')
+        Traceback (most recent call last):
+        ...
+        azul.uuids.InvalidUUIDPrefixError: '//foo.edu' is not a valid UUID prefix.
+        """
+
+        # FIXME: Move parsing of prefix to SourceSpec
+        #        https://github.com/DataBiosphere/azul/issues/3073
+        name, sep, prefix = spec.rpartition(':')
+        reject(sep == '',
+               'Source specifications must end in a colon followed by an optional UUID prefix')
+        return cls(prefix=prefix, name=name)
 
     def __str__(self) -> str:
-        return self.name
+        """
+        >>> str(SimpleSourceSpec(prefix='12', name='foo:bar/baz'))
+        'foo:bar/baz:12'
+        """
+        return f'{self.name}:{self.prefix}'
 
 
 SOURCE_REF = TypeVar('SOURCE_REF', bound='SourceRef')
 
 
 @attr.s(auto_attribs=True, frozen=True, kw_only=True)
-class SourceRef(Generic[SOURCE_NAME, SOURCE_REF]):
+class SourceRef(Generic[SOURCE_SPEC, SOURCE_REF]):
     """
     A reference to a repository source containing bundles to index. A repository
     has at least one source. A source is primarily referenced by its ID but we
@@ -112,40 +148,40 @@ class SourceRef(Generic[SOURCE_NAME, SOURCE_REF]):
     body is empty.
     """
     id: str
-    name: SOURCE_NAME
+    spec: SOURCE_SPEC
 
     _lookup: ClassVar[Dict[Tuple[Type['SourceRef'], str], 'SourceRef']] = {}
     _lookup_lock = RLock()
 
-    def __new__(cls: Type[SOURCE_REF], *, id: str, name: SOURCE_NAME) -> SOURCE_REF:
+    def __new__(cls: Type[SOURCE_REF], *, id: str, spec: SOURCE_SPEC) -> SOURCE_REF:
         """
         Interns instances by their ID and ensures that names are unambiguous
         for any given ID. Two different sources may still use the same name.
 
         >>> class S(SourceRef): pass
-        >>> a, b  = SimpleSourceName.parse('a'), SimpleSourceName.parse('b')
+        >>> a, b  = SimpleSourceSpec.parse('a:'), SimpleSourceSpec.parse('b:')
 
-        >>> S(id='1', name=a) is S(id='1', name=a)
+        >>> S(id='1', spec=a) is S(id='1', spec=a)
         True
 
-        >>> S(id='1', name=a) is S(id='2', name=a)
+        >>> S(id='1', spec=a) is S(id='2', spec=a)
         False
 
-        >>> S(id='1', name=b) # doctest: +NORMALIZE_WHITESPACE
+        >>> S(id='1', spec=b) # doctest: +NORMALIZE_WHITESPACE
         Traceback (most recent call last):
         ...
-        azul.RequirementError: ('Ambiguous source names for same ID.',
-                                SimpleSourceName(prefix='', name='a'),
-                                SimpleSourceName(prefix='', name='b'),
+        azul.RequirementError: ('Ambiguous source specs for same ID.',
+                                SimpleSourceSpec(prefix='', name='a'),
+                                SimpleSourceSpec(prefix='', name='b'),
                                 '1')
 
         Interning is done per class:
 
         >>> class T(S): pass
-        >>> T(id='1', name=a) is S(id='1', name=a)
+        >>> T(id='1', spec=a) is S(id='1', spec=a)
         False
 
-        >>> T(id='1', name=a) == S(id='1', name=a)
+        >>> T(id='1', spec=a) == S(id='1', spec=a)
         False
         """
         with cls._lookup_lock:
@@ -155,16 +191,16 @@ def __new__(cls: Type[SOURCE_REF], *, id: str, name: SOURCE_NAME) -> SOURCE_REF:
             except KeyError:
                 self = super().__new__(cls)
                 # noinspection PyArgumentList
-                self.__init__(id=id, name=name)
+                self.__init__(id=id, spec=spec)
                 lookup[cls, id] = self
             else:
                 assert self.id == id
-                require(self.name == name,
-                        'Ambiguous source names for same ID.', self.name, name, id)
+                require(self.spec == spec,
+                        'Ambiguous source specs for same ID.', self.spec, spec, id)
             return self
 
     def to_json(self):
-        return dict(id=self.id, name=str(self.name))
+        return dict(id=self.id, spec=str(self.spec))
 
 
 @attr.s(auto_attribs=True, frozen=True, kw_only=True, order=True)

src/azul/indexer/document.py

@@ -5,7 +5,6 @@
 from dataclasses import (
     dataclass,
     field,
-    fields,
 )
 from enum import (
     Enum,
@@ -38,7 +37,7 @@
 )
 from azul.indexer import (
     BundleFQID,
-    SimpleSourceName,
+    SimpleSourceSpec,
     SourceRef,
 )
 from azul.types import (
@@ -480,9 +479,6 @@ def to_json(self) -> JSON:
         return dict(entity_id=self.coordinates.entity.entity_id,
                     contents=self.contents)
 
-    def to_dict(self) -> JSON:
-        return {f.name: getattr(self, f.name) for f in fields(self)}
-
     @classmethod
     def _from_json(cls, document: JSON) -> Mapping[str, Any]:
         return {}
@@ -600,11 +596,11 @@ def delete(self):
         return False
 
 
-class DocumentSource(SourceRef[SimpleSourceName, SourceRef]):
+class DocumentSource(SourceRef[SimpleSourceSpec, SourceRef]):
 
     @classmethod
     def from_json(cls, source: JSON) -> 'DocumentSource':
-        return cls(id=source['id'], name=SimpleSourceName(name=source['name']))
+        return cls(id=source['id'], spec=SimpleSourceSpec.parse(source['spec']))
 
 
 @dataclass
@@ -668,7 +664,7 @@ class Aggregate(Document[AggregateCoordinates]):
     def __init__(self,
                  coordinates: AggregateCoordinates,
                  version: Optional[int],
-                 sources: Set[SourceRef[SimpleSourceName, SourceRef]],
+                 sources: Set[SourceRef[SimpleSourceSpec, SourceRef]],
                  contents: Optional[JSON],
                  bundles: Optional[List[JSON]],
                  num_contributions: int) -> None: ...
@@ -687,7 +683,7 @@ def field_types(cls, field_types: FieldTypes) -> FieldTypes:
             'num_contributions': pass_thru_int,
             'sources': {
                 'id': pass_thru_str,
-                'name': pass_thru_str
+                'spec': pass_thru_str
             },
             'bundles': {
                 'uuid': pass_thru_str,