Retrieve upstream tarballs from GitHub release assets

.upstream.d/10-SAGE_SERVER

@@ -0,0 +1,2 @@
+# When SAGE_SERVER is set, it should be an https/https server in the format of Sage mirrors.
+${SAGE_SERVER}/spkg/upstream/${SPKG}/

.upstream.d/20-gh.neting.cc-sagemath-sage-releases

@@ -0,0 +1,3 @@
+# Upstream packages as uploaded as GitHub release assets.
+# This file is automatically updated by the sage-update-version script.
+https://github.com/sagemath/sage/releases/download/10.1/

.upstream.d/30-www.sagemath.org-mirror_list

@@ -0,0 +1 @@
+https://www.sagemath.org/mirror_list

build/bin/write-dockerfile.sh

@@ -223,6 +223,7 @@ $ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.
 $ADD m4 ./m4
 $ADD pkgs pkgs
 $ADD build ./build
+$ADD .upstream.d ./.upstream.d
 ARG BOOTSTRAP=./bootstrap
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN
 

build/sage_bootstrap/download/mirror_list.py

@@ -19,7 +19,7 @@
 log = logging.getLogger()
 
 from sage_bootstrap.compat import urllib, urlparse
-from sage_bootstrap.env import SAGE_DISTFILES
+from sage_bootstrap.env import SAGE_DISTFILES, SAGE_ROOT
 
 from fcntl import flock, LOCK_SH, LOCK_EX
 from errno import ENOLCK
@@ -41,17 +41,66 @@ class MirrorListException(RuntimeError):
     pass
 
 
-MIRRORLIST_FILENAME = os.path.join(SAGE_DISTFILES, 'mirror_list')
+class MirrorList(object):
 
+    def __init__(self):
+        self.sources = []
+        upstream_d = os.path.join(SAGE_ROOT, '.upstream.d')
+        for fname in sorted(os.listdir(upstream_d)):
+            if '~' in fname or '#' in fname:
+                # Ignore auto-save and backup files
+                continue
+            try:
+                with open(os.path.join(upstream_d, fname), 'r') as f:
+                    for line in f:
+                        line = line.strip()
+                        if line.startswith('#'):
+                            continue
+                        if not line:
+                            continue
+                        line = line.replace('${SAGE_ROOT}', SAGE_ROOT)
+                        line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES)
+                        if '${SAGE_SERVER}' in line:
+                            SAGE_SERVER = os.environ.get("SAGE_SERVER", "")
+                            if not SAGE_SERVER:
+                                continue
+                            line = line.replace('${SAGE_SERVER}', SAGE_SERVER)
+                        if line.endswith('mirror_list'):
+                            cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2])
+                            self.sources.append(MirrorList_from_url(line, cache_filename))
+                        else:
+                            self.sources.append([line])
+            except IOError:
+                # Silently ignore files that do not exist
+                pass
 
-class MirrorList(object):
+    def __iter__(self):
+        """
+        Iterate through the list of mirrors.
+
+        This is the main entry point into the mirror list. Every
+        script should just use this function to try mirrors in order
+        of preference. This will not just yield the official mirrors,
+        but also urls for packages that are currently being tested.
+        """
+        for source in self.sources:
+            for mirror in source:
+                yield mirror
+
+
+class MirrorList_from_url(object):
 
-    URL = 'http://www.sagemath.org/mirror_list'
     MAXAGE = 24*60*60   # seconds
 
-    def __init__(self):
-        self.filename = MIRRORLIST_FILENAME
-        self.mirrors = None
+    def __init__(self, url, filename):
+        self.url = url
+        self.filename = filename
+        self._mirrors = None
+
+    @property
+    def mirrors(self):
+        if self._mirrors is not None:
+            return self._mirrors
 
         try:
             self.mirrorfile = open(self.filename, 'r+t')
@@ -67,8 +116,10 @@ def __init__(self):
                 # process while we waited for the lock?  Check again.
                 if self._must_refresh():
                     self._refresh()
-            if self.mirrors is None:
-                self.mirrors = self._load()
+            if self._mirrors is None:
+                self._mirrors = self._load()
+
+        return self._mirrors
 
     def _load(self, mirror_list=None):
         """
@@ -147,7 +198,7 @@ def _rank_mirrors(self):
             log.info('Cannot time mirrors via proxy, using default order')
         else:
             timed_mirrors.sort()
-            self.mirrors = [m[1] for m in timed_mirrors]
+            self._mirrors = [m[1] for m in timed_mirrors]
         log.info('Fastest mirror: ' + self.fastest)
 
     def _age(self):
@@ -176,12 +227,12 @@ def _refresh(self):
         """
         log.info('Downloading the Sage mirror list')
         try:
-            with contextlib.closing(urllib.urlopen(self.URL)) as f:
+            with contextlib.closing(urllib.urlopen(self.url)) as f:
                 mirror_list = f.read().decode("ascii")
         except IOError:
             log.critical('Downloading the mirror list failed, using cached version')
         else:
-            self.mirrors = self._load(mirror_list)
+            self._mirrors = self._load(mirror_list)
             self._rank_mirrors()
             self._save()
 
@@ -199,9 +250,9 @@ def __iter__(self):
         except KeyError:
             pass
         for mirror in self.mirrors:
-            yield mirror
-        # If all else fails: Try the packages we host ourselves
-        yield 'http://sagepad.org/'
+            if not mirror.endswith('/'):
+                mirror += '/'
+            yield mirror + '/'.join(['spkg', 'upstream', '${SPKG}'])
 
     @property
     def fastest(self):

build/sage_bootstrap/tarball.py

@@ -159,7 +159,10 @@ def download(self, allow_upstream=False):
         successful_download = False
         log.info('Attempting to download package {0} from mirrors'.format(self.filename))
         for mirror in MirrorList():
-            url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename])
+            url = mirror.replace('${SPKG}', self.package.name)
+            if not url.endswith('/'):
+                url += '/'
+            url += self.filename
             log.info(url)
             try:
                 Download(url, destination).run()

src/bin/sage-update-version

@@ -87,6 +87,23 @@ EOF
 # Create a top-level VERSION.txt file, which some external utilities rely on
 echo "$SAGE_VERSION_BANNER" > "$SAGE_ROOT/VERSION.txt"
 
+# Add version to the front of GitHub release assets URLs.
+SAGE_MINOR_VERSION=${SAGE_VERSION//.alpha*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.beta*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.dev*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.post*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.rc*/}
+( echo "https://github.com/sagemath/sage/releases/download/$SAGE_MINOR_VERSION/"
+  sed '/^#/d' "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases"
+) | uniq | head -n 3 > "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases.tmp"
+( cat <<EOF
+# Upstream packages as uploaded as GitHub release assets.
+# This file is automatically updated by the sage-update-version script.
+EOF
+  cat "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases.tmp"
+) > "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases"
+rm -f "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases.tmp"
+
 # Regenerate auto-generated files tarball
 "$SAGE_ROOT/bootstrap" -s
 
@@ -106,6 +123,7 @@ git commit -m "Updated SageMath version to $SAGE_VERSION" -- \
     "$SAGE_ROOT/build/pkgs/configure/package-version.txt" \
     "$SAGE_ROOT/build/pkgs/*/install-requires.txt" \
     "$SAGE_ROOT"/pkgs/*/VERSION.txt \
+    "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases" \
     || die "Error committing to the repository."
 
 git tag -a "$SAGE_VERSION" -m "$SAGE_VERSION_BANNER" \