Automate the creation of requirement.txt and fetch deps

etc/scripts/freeze_and_update_reqs.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+from fnmatch import fnmatchcase
+import os
+from subprocess import run
+from shutil import copy, rmtree
+import sys
+import tempfile
+
+from commoncode.fileutils import resource_iter
+
+python_version = str(sys.version_info[0]) + str(sys.version_info[1])
+py_abi = '{0}cp{1}{0}'.format('*', python_version)
+
+
+def generate_req_text(find_links, req_file, package_name=None, upgrade=False):
+    """
+    Generate a requirement file at `req_file` of all dependencies wheels and 
+    sdists present in the `input_dir` directory.If a `package_name` is provided 
+    it will be updated to its latest version.
+    """
+    thirdparty = resource_iter(find_links, with_dirs=False)
+    dependencies = [
+        files
+        for files in thirdparty
+        if fnmatchcase(files, '*py3*')
+        or fnmatchcase(files, py_abi)
+        or (
+            fnmatchcase(files, '*tar.gz*')
+            and not fnmatchcase(files, '*py2-ipaddress-3.4.1.tar.gz*')
+        )
+    ]
+    with tempfile.TemporaryDirectory() as temp_dir:
+        for deps in dependencies:
+            copy(deps, temp_dir)
+        pip_args = [
+            'pip-compile',
+            '--generate-hashes',
+            '--find-links',
+            temp_dir,
+            '--output-file',
+            req_file,
+            '--allow-unsafe',
+            '--pip-args',
+            '--no-index',
+        ]
+        if upgrade:
+            pip_args.append('--upgrade')
+        if package_name:
+            pip_args.extend(['--upgrade-package', package_name])
+        run(pip_args)
+
+
+def main_with_args(args: str) -> None:
+    parser = argparse.ArgumentParser(
+        description="""Generate a requirement file at `req_file` of all dependencies wheels 
+and sdists present in the `input_dir` directory.If a `package_name` is 
+provided it will be updated to its latest version.
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        '--find-links',
+        help='Required: Look for archives in this directory or on this HTML page',
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        '--requirement',
+        help='Required: Requirement file name.',
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        '--upgrade',
+        help='Optional: Try to upgrade all dependencies to their latest versions',
+        action='store_true',
+    )
+
+    parser.add_argument(
+        '--upgrade-package',
+        help='Optional: Specify particular packages to upgrade.',
+        type=str,
+        default=None,
+    )
+
+    args = parser.parse_args()
+
+    find_links = args.find_links
+    requirement = args.requirement
+    upgrade_package = args.upgrade_package or None
+    upgrade = args.upgrade or False
+    generate_req_text(find_links, requirement, upgrade_package, upgrade)
+
+
+def main() -> None:
+    main_with_args(sys.argv[1:])
+
+
+if __name__ == '__main__':
+    main()

etc/scripts/search-package.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import fnmatch
+import sys
+
+from commoncode.fileutils import resource_iter
+
+def search_package(package_name, target, version=None):
+    """
+    Search `package_name` (with an optional `version`) in the `target` directory.
+    Print results on screen.
+    """
+
+    if version:
+        package_name = '*{}-{}*'.format(package_name, version)
+    else:
+        package_name = '*{}*'.format(package_name)
+    thirdparty = resource_iter(target, with_dirs=False)
+    dependency = [
+        files for files in thirdparty if fnmatch.fnmatchcase(files, package_name)
+    ]
+    if dependency:
+        whl = [files for files in dependency if files.endswith('.whl')]
+        ## There are multiple version of same package So list of wheel will be considered.
+        sdist = [files for files in dependency if files.endswith('.tar.gz')]
+        about = [files for files in dependency if files.endswith('.ABOUT')]
+        notice = [files for files in dependency if files.endswith('.NOTICE')]
+        license = [files for files in dependency if files.endswith('.LICENSE')]
+        print('\nSearched package wheel are:')
+        print(*whl, sep='\n')
+        if sdist:
+            print('\nCorresponding sdist are:')
+            print(*sdist, sep='\n')
+        else:
+            print('\nCorresponding sdist does not exits in target')
+        if about:
+            print('\nCorresponding .ABOUT are:')
+            print(*about, sep='\n')
+        else:
+            print('\nCorresponding .ABOUT does not exits in target\n')
+        if license:
+            print('\nCorresponding .LICENSE are:')
+            print(*licence, sep='\n')
+        else:
+            print('\nCorresponding .LICENSE does not exits in target')
+        if notice:
+            print('\nCorresponding .NOTICE are:')
+            print(*notice, sep='\n')
+        else:
+            print('\nCorresponding .NOTICE does not exits in target\n')
+    else:
+        print('\nSpecified package does not exist\n')
+
+
+def main_with_args(args: str) -> None:
+    parser = argparse.ArgumentParser(
+        description="""Search PACKAGE_NAME (with an optional VERSION_OF_PACKAGE) in the
+        TARGET_DIR directory.Print results on screen.
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        '--package_name',
+        help='Name of the package to search in the TARGET_DIR directory',
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        '--directory',
+        help='Directory to search for package wheels and tarballs. [default: thirdparty]',
+        type=str,
+        default='thirdparty',
+    )
+
+    parser.add_argument(
+        '--version', 
+        help='Optional package version to search.', 
+        type=str, 
+        default=None,
+    )
+
+    args = parser.parse_args()
+
+    package_name = args.package_name
+    directory = args.directory
+    version = args.version
+    search_package(package_name, directory, version)
+
+
+def main() -> None:
+    main_with_args(sys.argv[1:])
+
+
+if __name__ == '__main__':
+    main()