Add minimal files to Package model #1554

This is optional and not on by default.
Also removed Maven-only test code from base model

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

src/packagedcode/models.py

@@ -300,19 +300,22 @@ def set_purl(self, package_url):
             if not self_val and purl_val:
                 setattr(self, att, purl_val)
 
-    def to_dict(self, **kwargs):
+    def to_dict(self, with_files=False, **kwargs):
         """
         Return an OrderedDict of primitive Python types.
         """
         mapping = attr.asdict(self, dict_factory=OrderedDict)
         if self.qualifiers:
             mapping['qualifiers'] = normalize_qualifiers(self.qualifiers, encode=True)
 
-        if not kwargs.get('exclude_properties'):
-            mapping['purl'] = self.purl
-            mapping['repository_homepage_url'] = self.repository_homepage_url()
-            mapping['repository_download_url'] = self.repository_download_url()
-            mapping['api_data_url'] = self.api_data_url()
+        mapping['purl'] = self.purl
+        mapping['repository_homepage_url'] = self.repository_homepage_url()
+        mapping['repository_download_url'] = self.repository_download_url()
+        mapping['api_data_url'] = self.api_data_url()
+
+        if not with_files:
+            mapping.pop('files')
+
         return mapping
 
     @classmethod
@@ -366,6 +369,22 @@ class DependentPackage(BaseModel):
              'exact version.')
 
 
+@attr.s()
+class Resource(BaseModel):
+    """
+    A resource represent a file or directory with minimal information.
+    """
+    path = String(
+        repr=True,
+        label='path',
+        help='The file or directory POSIX path decoded as unicode using the filesystem encoding.')
+
+    type = String(
+        repr=True,
+        label='type',
+        help='The resource type: either a "file" or a "directory"')
+
+
 @attr.s()
 class Package(BasePackage):
     """
@@ -481,6 +500,11 @@ class Package(BasePackage):
              'this package. For instance an SRPM is the "source package" for a '
              'binary RPM.')
 
+    files = List(
+        item_type=Resource,
+        label='List of files and directories contained in this package',
+        help='List of files and directories contained in this package.')
+
     def __attrs_post_init__(self, *args, **kwargs):
         if not self.type and hasattr(self, 'default_type'):
             self.type = self.default_type

tests/packagedcode/test_maven.py

@@ -214,9 +214,19 @@ def test_parse_to_package_and_validate(self):
     def test_parse_to_package_then_back(self):
         test_file = self.get_test_loc('maven_misc/spring-beans-4.2.2.RELEASE.pom.xml')
         package = maven.parse(test_file)
-        package2 = maven.MavenPomPackage(**package.to_dict(exclude_properties=True))
+        packaged = package.to_dict()
+        del packaged['purl']
+        del packaged['repository_homepage_url']
+        del packaged['repository_download_url']
+        del packaged['api_data_url']
+
+        package2 = maven.MavenPomPackage(**packaged)
         assert package.to_dict().items() == package2.to_dict().items()
 
+        # create ignore unknown fields
+        package3 = maven.MavenPomPackage.create(**package.to_dict())
+        assert package.to_dict().items() == package3.to_dict().items()
+
     def test_package_root_is_properly_returned_for_metainf_poms(self):
         test_dir = self.get_test_loc('maven_misc/package_root')
         resource_attributes = dict(packages=attr.ib(default=attr.Factory(list), repr=False))

tests/packagedcode/test_package_models.py

@@ -105,6 +105,24 @@ def test_Package_model_qualifiers_are_serialized_as_strings(self):
         )
         assert 'this=that' == package.to_dict()['qualifiers']
 
+    def test_Package_model_qualifiers_are_serialized_normalized(self):
+        package = models.Package(
+            type='maven',
+            name='this',
+            version='23',
+            qualifiers=OrderedDict(this='that', abc='asdasdasd//3#?.')
+        )
+        assert 'abc=asdasdasd//3%23%3F.&this=that' == package.to_dict()['qualifiers']
+
+    def test_Package_model_files_are_serialized_optionally(self):
+        package = models.Package(
+            type='maven',
+            name='this',
+            version='23',)
+        assert [] == package.to_dict(with_files=True)['files']
+        package.files.append(models.Resource(type='file', path='some/path'))
+        assert [OrderedDict([('path', u'some/path'), ('type', u'file')])] == package.to_dict(with_files=True)['files']
+
     def test_Package_full(self):
         package = Package(
             type='rpm',