Add support for relative URLs in repository.json

Fixes #1329

This commit enables RepositoryProvider to resolve relative URLs in
- package-level `details` keys
- release-level `base` and `url` keys

using the repository's location as root.

Example:

A repository https://github.com/user/repository/repository.json with
following content:

{
   "name": "my-package",
   "details": "../my-package"
   "releases": [
      {
         "url": "./downloads/my-package.sublime-package"
      }
   ]
}

is resolved to...

{
   "name": "my-package",
   "details": "https://github.com/user/my-package"
   "releases": [
      {
         "url": "https://github.com/user/repository/downloads/my-package.sublime-package"
      }
   ]
}

example-repository.json

@@ -394,6 +394,32 @@
 					// SHOULD be UTC
 					"date": "2011-09-18 20:12:41",
 
+					// The obligatory version selector
+					"sublime_text": "*"
+				}
+			]
+		},
+
+		// URLs may be provided relative to the location of this repository.json
+		{
+			"details": "../alignment",
+			"releases": [
+				{
+					// The version number needs to be a semantic version
+					// number per http://semver.org 2.x.x
+					"version": "2.0.0",
+
+					// The URL needs to be a zip file containing the package.
+					// It is permissible for the zip file to contain a single
+					// root folder with any name. All file will be extracted
+					// out of this single root folder. This allows zip files
+					// from GitHub and BitBucket to be used a sources.
+					"url": "./downloads/alignment-2.0.0.zip",
+
+					// The date MUST be in the form "YYYY-MM-DD HH:MM:SS" and
+					// SHOULD be UTC
+					"date": "2011-09-18 20:12:41",
+
 					// The obligatory version selector
 					"sublime_text": "*"
 				}

package_control/download_manager.py

@@ -185,6 +185,41 @@ def resolve_urls(root_url, uris):
         yield url
 
 
+def resolve_url(root_url, url):
+    """
+    Convert a list of relative uri's to absolute urls/paths.
+
+    :param root_url:
+        The root url string
+
+    :param uris:
+        An iteratable of relative uri's to resolve.
+
+    :returns:
+        A generator of resolved URLs
+    """
+
+    scheme_match = re.match(r'(https?:)//', root_url, re.I)
+    if scheme_match is None:
+        root_dir = os.path.dirname(root_url)
+    else:
+        root_dir = ''
+
+    if url.startswith('//'):
+        if scheme_match is not None:
+            return scheme_match.group(1) + url
+        else:
+            return 'https:' + url
+
+    elif url.startswith('./') or url.startswith('../'):
+        if root_dir:
+            return os.path.normpath(os.path.join(root_dir, url))
+        else:
+            return urljoin(root_url, url)
+
+    return url
+
+
 def update_url(url, debug):
     """
     Takes an old, out-dated URL and updates it. Mostly used with GitHub URLs

package_control/providers/repository_provider.py

@@ -10,7 +10,7 @@
 from ..clients.github_client import GitHubClient
 from ..clients.gitlab_client import GitLabClient
 from ..console_write import console_write
-from ..download_manager import http_get, resolve_urls, update_url
+from ..download_manager import http_get, resolve_url, resolve_urls, update_url
 from ..downloaders.downloader_exception import DownloaderException
 from ..package_version import version_sort
 from .base_repository_provider import BaseRepositoryProvider
@@ -310,7 +310,7 @@ def assert_release_keys(download_info):
                     # Validate url
                     value = release.get('url')
                     if value:
-                        download_info['url'] = update_url(value, debug)
+                        download_info['url'] = update_url(resolve_url(self.repo_url, value), debug)
 
                     # Validate supported platforms
                     value = release.get('platforms', ['*'])
@@ -347,19 +347,20 @@ def assert_release_keys(download_info):
                                 (info['name'], self.repo_url)
                             ))
 
+                        base_url = resolve_url(self.repo_url, base)
                         downloads = None
 
                         if tags:
                             extra = None
                             if tags is not True:
                                 extra = tags
                             for client in clients:
-                                downloads = client.download_info_from_tags(base, extra)
+                                downloads = client.download_info_from_tags(base_url, extra)
                                 if downloads is not None:
                                     break
                         else:
                             for client in clients:
-                                downloads = client.download_info_from_branch(base, branch)
+                                downloads = client.download_info_from_branch(base_url, branch)
                                 if downloads is not None:
                                     break
 
@@ -501,7 +502,7 @@ def get_packages(self, invalid_sources=None):
                 if package.get(field):
                     info[field] = package.get(field)
 
-            details = package.get('details')
+            details = resolve_url(self.repo_url, package.get('details'))
             releases = package.get('releases')
 
             # Try to grab package-level details from GitHub or BitBucket
@@ -585,7 +586,7 @@ def get_packages(self, invalid_sources=None):
                     if field in release:
                         value = release[field]
                         if field == 'url':
-                            value = update_url(value, debug)
+                            value = update_url(resolve_url(self.repo_url, value), debug)
                         if field == 'platforms' and not isinstance(release['platforms'], list):
                             value = [value]
                         download_info[field] = value
@@ -612,7 +613,7 @@ def get_packages(self, invalid_sources=None):
                         download_info['sublime_text'] = '<3000'
 
                     if 'details' in release:
-                        download_details = release['details']
+                        download_details = resolve_url(self.repo_url, release['details'])
 
                         try:
                             downloads = None
@@ -672,19 +673,20 @@ def get_packages(self, invalid_sources=None):
                                     (info['name'], self.repo_url)
                                 ))
 
+                            base_url = resolve_url(self.repo_url, base)
                             downloads = None
 
                             if tags:
                                 extra = None
                                 if tags is not True:
                                     extra = tags
                                 for client in clients:
-                                    downloads = client.download_info_from_tags(base, extra)
+                                    downloads = client.download_info_from_tags(base_url, extra)
                                     if downloads is not None:
                                         break
                             else:
                                 for client in clients:
-                                    downloads = client.download_info_from_branch(base, branch)
+                                    downloads = client.download_info_from_branch(base_url, branch)
                                     if downloads is not None:
                                         break
 
@@ -763,7 +765,7 @@ def has_broken_release():
                     info[field] = []
 
             if 'readme' in info:
-                info['readme'] = update_url(info['readme'], debug)
+                info['readme'] = update_url(resolve_url(self.repo_url, info['readme']), debug)
 
             for field in ['description', 'readme', 'issues', 'donate', 'buy']:
                 if field not in info:

package_control/tests/__init__.py

@@ -6,6 +6,7 @@
 
 TEST_CLASSES = [
     package_version.PackageVersionTests,
+    downloaders.ResolveUrlTests,
     downloaders.CurlDownloaderTests,
     downloaders.OscryptoDownloaderTests,
     downloaders.UrlLibDownloaderTests,

package_control/tests/downloaders.py

@@ -1,13 +1,53 @@
 import unittest
 
 from .. import downloaders
+from ..download_manager import resolve_url
 from ..downloaders.binary_not_found_error import BinaryNotFoundError
 from ..downloaders.downloader_exception import DownloaderException
 from ..http_cache import HttpCache
 
 from ._config import USER_AGENT, DEBUG, GH_USER, GH_PASS
 
 
+class ResolveUrlTests(unittest.TestCase):
+
+    def test_match_absolute_url(self):
+        self.assertEqual(
+            "https://github.com/packagecontrol-test-2/package_control-tester-2",
+            resolve_url(
+                "https://github.com/packagecontrol-test/package_control-tester/repository.json",
+                "https://github.com/packagecontrol-test-2/package_control-tester-2"
+            )
+        )
+
+    def test_match_absolute_same_scheme_url(self):
+        self.assertEqual(
+            "https://github.com/packagecontrol-test-2/package_control-tester-2",
+            resolve_url(
+                "https://github.com/packagecontrol-test/package_control-tester/repository.json",
+                "//github.com/packagecontrol-test-2/package_control-tester-2"
+            )
+        )
+
+    def test_match_relative_sibling_url(self):
+        self.assertEqual(
+            "https://github.com/packagecontrol-test/package_control-tester-2",
+            resolve_url(
+                "https://github.com/packagecontrol-test/package_control-tester/repository.json",
+                "../package_control-tester-2"
+            )
+        )
+
+    def test_match_relative_child_url(self):
+        self.assertEqual(
+            "https://github.com/packagecontrol-test/package_control-tester/issues",
+            resolve_url(
+                "https://github.com/packagecontrol-test/package_control-tester/repository.json",
+                "./issues"
+            )
+        )
+
+
 class DownloaderTestsMixin:
 
     def downloader(self, cache_length=604800):

sublime-package.json

@@ -487,9 +487,9 @@
 						"InfoValues": {
 							"DetailsValue": {
 								"type": "string",
-								"format": "uri",
-								"markdownDescription": "URL of a BitBucket/Github/Gitlab repository to fetch information about the package or library from.",
-								"pattern": "https?://(?:bitbucket\\.org|github\\.com|gitlab\\.com)/[^/#?]+/[^/#?]+"
+								"format": "uri-reference",
+								"markdownDescription": "URL of a BitBucket/Github/Gitlab repository to fetch information about the package or library from.\n\nCan be a relative path to the location of this repository.json (e.g.: `../mypackage`).",
+								"pattern": "(?:https?://(?:bitbucket\\.org|github\\.com|gitlab\\.com)/[^/#?]+/[^/#?]+|\\.\\.?/.*)"
 							},
 							"NameValue": {
 								"type": "string",
@@ -624,8 +624,8 @@
 							},
 							"url": {
 								"type": "string",
-								"format": "uri",
-								"markdownDescription": "```json\n\"url\": \"https://any-domain.com/package.sublime-package\"\n```\n\nURL of the download artefact. Can be a `*.sublime-package` or a `*.whl`\n\n_Used in conjunction with `version`._"
+								"format": "uri-reference",
+								"markdownDescription": "```json\n\"url\": \"https://any-domain.com/downloads/mypackage-2.0.0.zip\"\n```\n\nURL of the download artefact.\n\nThe URL needs to be a zip file containing the package.\n\nIt is permissible for the zip file to contain a single root folder with any name. All file will be extracted out of this single root folder. This allows zip files from GitHub and BitBucket to be used a sources.\n\nThe URL can be relative to the location of this repository.json (e.g.: `./downloads/mypackage-2.0.0.zip`).\n\n_Used in conjunction with `version`._"
 							},
 							"version": {
 								"type": "string",