Collect dpkg and rpm source pkg info

This PR adds source package name and source package version
information to the Package object data model. It also adds scripts
in base.yml for rpm and dpkg package managers to collect source
package names and versions.

Tern currently reports binary package metadata in its reports. Source
packages exist in operating systems like Debian and RedHat and differ
from binary packages. Source packages provide all of the necessary files
to compile or build a desired piece of software. Binary packages are
what get produced as a result of building a source package and are
what typically gets installed in an environment. Binary packages can
have different names and/or versions as their source package.

Source packages are relevant in the context of security scanning as most
CVEs are reported by source package name and version.

Resolves #1083

Signed-off-by: Rose Judge <rjudge@vmware.com>

tern/analyze/default/bundle.py

@@ -50,7 +50,9 @@ def convert_to_pkg_dicts(attr_lists):
                'copyright': 'copyrights',
                'proj_url': 'proj_urls',
                'pkg_licenses': 'pkg_licenses',
-               'files': 'files'}
+               'files': 'files',
+               'src_name': 'source_names',
+               'src_version': 'source_versions'}
     pkg_list = []
     len_names = len(attr_lists['names'])
     # make a list of keys that correspond with package property names

tern/analyze/default/command_lib/base.yml

@@ -88,6 +88,20 @@ dpkg:
         container:
           - "dpkg-query -W -f '${Version}\n'"
     delimiter: "\n"
+  source_names:
+    invoke:
+      1:
+        container:
+          - "pkgs=`dpkg-query -W -f '${Package}\n'`"
+          - "for p in $pkgs; do dpkg-query -f '${{source:Package}}\n' -W $p; done"
+    delimiter: "\n"
+  source_versions:
+    invoke:
+      1:
+        container:
+          - "pkgs=`dpkg-query -W -f '${Package}\n'`"
+          - "for p in $pkgs; do dpkg-query -f '${{source:Version}}\n' -W $p; done"
+    delimiter: "\n"
   copyrights:
     invoke:
       1:
@@ -229,6 +243,22 @@ rpm:
         container:
           - 'rpm --query --all --queryformat "%{license}\n" 2>/dev/null'
     delimiter: "\n"
+  source_names:
+    invoke:
+      1:
+        container:
+          - "pkgs=`rpm --query --all --queryformat \"%{name}\n\" 2>/dev/null`"
+          - |
+            for p in $pkgs; do rpm -q --qf "%{SOURCERPM}\n" "$p" | sed 's/-[0-9].*\.src\.rpm$//'; done
+    delimiter: "\n"
+  source_versions:
+    invoke:
+      1:
+        container:
+          - "pkgs=`rpm --query --all --queryformat \"%{name}\n\" 2>/dev/null`"
+          - |
+            for p in $pkgs; do srcrpm="$(rpm -q --qf "%{SOURCERPM}\n" "$p")"; name="$( echo "${srcrpm}" | sed 's/-[0-9].*\.src\.rpm$//' )";version="$( echo "${srcrpm}" | sed "s/^${name}-\(.*\)\..*\.src\.rpm\$/\1/" )";echo "${version}"; done
+    delimiter: "\n"
   files:
     invoke:
       1:

tern/analyze/default/command_lib/command_lib.py

@@ -36,10 +36,10 @@
 with open(os.path.abspath(snippet_file), encoding='utf-8') as f:
     command_lib['snippets'] = yaml.safe_load(f)
 # list of package information keys that the command library can accomodate
-base_keys = {'names', 'versions', 'licenses', 'copyrights', 'proj_urls',
-             'srcs', 'files'}
-package_keys = {'name', 'version', 'license', 'copyright', 'proj_url', 'src',
-                'files'}
+base_keys = {'names', 'versions', 'licenses', 'source_names',
+             'source_versions', 'copyrights', 'proj_urls', 'srcs', 'files'}
+package_keys = {'name', 'version', 'license', 'src_name', 'src_version',
+                'copyright', 'proj_url', 'src', 'files'}
 
 # global logger
 logger = logging.getLogger(constants.logger_name)

tern/classes/package.py

@@ -22,6 +22,8 @@ class Package:
         checksum: checksum as package property
         files: list of files in a package
         pkg_licenses: all licenses found within a package
+        src_name: the source package associated with the binary package
+        src_version: the source package version
 
     methods:
         to_dict: returns a dict representation of the instance
@@ -43,6 +45,8 @@ def __init__(self, name):
         self.__files = []
         self.__pkg_licenses = []
         self.__pkg_format = ''
+        self.__src_name = ''
+        self.__src_version = ''
 
     @property
     def name(self):
@@ -124,6 +128,22 @@ def pkg_format(self):
     def pkg_format(self, pkg_format):
         self.__pkg_format = pkg_format
 
+    @property
+    def src_name(self):
+        return self.__src_name
+
+    @src_name.setter
+    def src_name(self, src_name):
+        self.__src_name = src_name
+
+    @property
+    def src_version(self):
+        return self.__src_version
+
+    @src_version.setter
+    def src_version(self, src_version):
+        self.__src_version = src_version
+
     def get_file_paths(self):
         """Return a list of paths of all the files in a package"""
         return [f.path for f in self.__files]
@@ -195,6 +215,8 @@ def fill(self, package_dict):
             copyright: <package copyright text>
             proj_url: <project url>
             files: <package files>
+            src_name: <source package>
+            src_ver: <source package version>
         the way to use this method is to instantiate the class with the
         name and then give it a package dictionary to fill in the rest
         return true if package name is the same as the one used to instantiate

tests/test_class_package.py

@@ -23,6 +23,8 @@ def setUp(self):
         self.p1.checksum = '123abc456'
         self.p1.pkg_licenses = ['MIT', 'GPL']
         self.p1.pkg_format = 'deb'
+        self.p1.src_name = 'p1src'
+        self.p1.src_version = '1.0'
 
         self.p2 = Package('p2')
 
@@ -74,6 +76,8 @@ def testGetters(self):
         self.assertEqual(self.p1.checksum, '123abc456')
         self.assertEqual(self.p1.pkg_licenses, ['MIT', 'GPL'])
         self.assertEqual(self.p1.pkg_format, 'deb')
+        self.assertEqual(self.p1.src_name, 'p1src')
+        self.assertEqual(self.p1.src_version, '1.0')
 
     def testAddFile(self):
         p1 = Package('package')
@@ -124,6 +128,8 @@ def testToDict(self):
         self.assertEqual(a_dict['files'][0]['path'], 'abc/pqr/test.java')
         self.assertEqual(a_dict['pkg_licenses'], ['MIT', 'GPL'])
         self.assertEqual(a_dict['pkg_format'], 'deb')
+        self.assertEqual(a_dict['src_name'], 'p1src')
+        self.assertEqual(a_dict['src_version'], '1.0')
 
     def testToDictTemplate(self):
         template1 = TestTemplate1()
@@ -167,7 +173,10 @@ def testFill(self):
                   'pkg_licenses': ['MIT', 'GPL'],
                   'files': [{'name': 'a.txt', 'path': '/usr/a.txt'},
                             {'name': 'b.txt', 'path': '/lib/b.txt'}],
-                  'pkg_format': 'rpm'}
+                  'pkg_format': 'rpm',
+                  'src_name': 'p1src',
+                  'src_version': '1.0'
+                  }
         p = Package('p1')
         p.fill(p_dict)
         self.assertEqual(p.name, 'p1')