gh-36435: Retrieve upstream tarballs from GitHub release assets

    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes #1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->
We propose to add GitHub release assets as a source from which upstream
tarballs can be retrieved. This would take priority over trying the Sage
mirrors, thus reducing the reliance of the project on self-maintained
file servers. https://docs.github.com/en/repositories/releasing-
projects-on-github/about-releases#storage-and-bandwidth-quotas

For the purpose of testing this PR, one tarball missing on the mirrors
(#36381 (comment))
has been uploaded to https://github.com/sagemath/sage/releases/tag/10.1

Creating GitHub releases and depositing the upstream tarballs can later
be automated in a GH Actions workflow, e.g., using
https://github.com/softprops/action-gh-release. As we don't make GitHub
releases for beta versions, the previous stable release can be used.

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #36435
Reported by: Matthias Köppe
Reviewer(s): Dima Pasechnik, John H. Palmieri, Matthias Köppe

.upstream.d/10-SAGE_SERVER

@@ -0,0 +1,2 @@
+# When SAGE_SERVER is set, it should be an https/https server in the format of Sage mirrors.
+${SAGE_SERVER}/spkg/upstream/${SPKG}/

.upstream.d/20-gh.neting.cc-sagemath-sage-releases

@@ -0,0 +1,3 @@
+# Upstream packages as uploaded as GitHub release assets.
+# This file is automatically updated by the sage-update-version script.
+https://github.com/sagemath/sage/releases/download/10.1/

.upstream.d/30-www.sagemath.org-mirror_list

@@ -0,0 +1 @@
+https://www.sagemath.org/mirror_list

build/bin/write-dockerfile.sh

@@ -226,6 +226,7 @@ $ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.
 $ADD m4 ./m4
 $ADD pkgs pkgs
 $ADD build ./build
+$ADD .upstream.d ./.upstream.d
 ARG BOOTSTRAP=./bootstrap
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN
 

build/sage_bootstrap/download/mirror_list.py

@@ -19,7 +19,7 @@
 log = logging.getLogger()
 
 from sage_bootstrap.compat import urllib, urlparse
-from sage_bootstrap.env import SAGE_DISTFILES
+from sage_bootstrap.env import SAGE_DISTFILES, SAGE_ROOT
 
 from fcntl import flock, LOCK_SH, LOCK_EX
 from errno import ENOLCK
@@ -41,17 +41,66 @@ class MirrorListException(RuntimeError):
     pass
 
 
-MIRRORLIST_FILENAME = os.path.join(SAGE_DISTFILES, 'mirror_list')
+class MirrorList(object):
 
+    def __init__(self):
+        self.sources = []
+        upstream_d = os.path.join(SAGE_ROOT, '.upstream.d')
+        for fname in sorted(os.listdir(upstream_d)):
+            if '~' in fname or '#' in fname:
+                # Ignore auto-save and backup files
+                continue
+            try:
+                with open(os.path.join(upstream_d, fname), 'r') as f:
+                    for line in f:
+                        line = line.strip()
+                        if line.startswith('#'):
+                            continue
+                        if not line:
+                            continue
+                        line = line.replace('${SAGE_ROOT}', SAGE_ROOT)
+                        line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES)
+                        if '${SAGE_SERVER}' in line:
+                            SAGE_SERVER = os.environ.get("SAGE_SERVER", "")
+                            if not SAGE_SERVER:
+                                continue
+                            line = line.replace('${SAGE_SERVER}', SAGE_SERVER)
+                        if line.endswith('mirror_list'):
+                            cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2])
+                            self.sources.append(MirrorList_from_url(line, cache_filename))
+                        else:
+                            self.sources.append([line])
+            except IOError:
+                # Silently ignore files that do not exist
+                pass
 
-class MirrorList(object):
+    def __iter__(self):
+        """
+        Iterate through the list of mirrors.
+
+        This is the main entry point into the mirror list. Every
+        script should just use this function to try mirrors in order
+        of preference. This will not just yield the official mirrors,
+        but also urls for packages that are currently being tested.
+        """
+        for source in self.sources:
+            for mirror in source:
+                yield mirror
+
+
+class MirrorList_from_url(object):
 
-    URL = 'http://www.sagemath.org/mirror_list'
     MAXAGE = 24*60*60   # seconds
 
-    def __init__(self):
-        self.filename = MIRRORLIST_FILENAME
-        self.mirrors = None
+    def __init__(self, url, filename):
+        self.url = url
+        self.filename = filename
+        self._mirrors = None
+
+    @property
+    def mirrors(self):
+        if self._mirrors is not None:
+            return self._mirrors
 
         try:
             self.mirrorfile = open(self.filename, 'r+t')
@@ -67,8 +116,10 @@ def __init__(self):
                 # process while we waited for the lock?  Check again.
                 if self._must_refresh():
                     self._refresh()
-            if self.mirrors is None:
-                self.mirrors = self._load()
+            if self._mirrors is None:
+                self._mirrors = self._load()
+
+        return self._mirrors
 
     def _load(self, mirror_list=None):
         """
@@ -147,7 +198,7 @@ def _rank_mirrors(self):
             log.info('Cannot time mirrors via proxy, using default order')
         else:
             timed_mirrors.sort()
-            self.mirrors = [m[1] for m in timed_mirrors]
+            self._mirrors = [m[1] for m in timed_mirrors]
         log.info('Fastest mirror: ' + self.fastest)
 
     def _age(self):
@@ -176,12 +227,12 @@ def _refresh(self):
         """
         log.info('Downloading the Sage mirror list')
         try:
-            with contextlib.closing(urllib.urlopen(self.URL)) as f:
+            with contextlib.closing(urllib.urlopen(self.url)) as f:
                 mirror_list = f.read().decode("ascii")
         except IOError:
             log.critical('Downloading the mirror list failed, using cached version')
         else:
-            self.mirrors = self._load(mirror_list)
+            self._mirrors = self._load(mirror_list)
             self._rank_mirrors()
             self._save()
 
@@ -199,9 +250,9 @@ def __iter__(self):
         except KeyError:
             pass
         for mirror in self.mirrors:
-            yield mirror
-        # If all else fails: Try the packages we host ourselves
-        yield 'http://sagepad.org/'
+            if not mirror.endswith('/'):
+                mirror += '/'
+            yield mirror + '/'.join(['spkg', 'upstream', '${SPKG}'])
 
     @property
     def fastest(self):

build/sage_bootstrap/tarball.py

@@ -159,7 +159,10 @@ def download(self, allow_upstream=False):
         successful_download = False
         log.info('Attempting to download package {0} from mirrors'.format(self.filename))
         for mirror in MirrorList():
-            url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename])
+            url = mirror.replace('${SPKG}', self.package.name)
+            if not url.endswith('/'):
+                url += '/'
+            url += self.filename
             log.info(url)
             try:
                 Download(url, destination).run()

src/bin/sage-update-version

@@ -87,6 +87,23 @@ EOF
 # Create a top-level VERSION.txt file, which some external utilities rely on
 echo "$SAGE_VERSION_BANNER" > "$SAGE_ROOT/VERSION.txt"
 
+# Add version to the front of GitHub release assets URLs.
+SAGE_MINOR_VERSION=${SAGE_VERSION//.alpha*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.beta*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.dev*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.post*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.rc*/}
+( echo "https://github.com/sagemath/sage/releases/download/$SAGE_MINOR_VERSION/"
+  sed '/^#/d' "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases"
+) | uniq | head -n 3 > "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases.tmp"
+( cat <<EOF
+# Upstream packages as uploaded as GitHub release assets.
+# This file is automatically updated by the sage-update-version script.
+EOF
+  cat "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases.tmp"
+) > "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases"
+rm -f "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases.tmp"
+
 # Regenerate auto-generated files tarball
 "$SAGE_ROOT/bootstrap" -s
 
@@ -106,6 +123,7 @@ git commit -m "Updated SageMath version to $SAGE_VERSION" -- \
     "$SAGE_ROOT/build/pkgs/configure/package-version.txt" \
     "$SAGE_ROOT/build/pkgs/*/install-requires.txt" \
     "$SAGE_ROOT"/pkgs/*/VERSION.txt \
+    "$SAGE_ROOT/.upstream.d/20-gh.neting.cc-sagemath-sage-releases" \
     || die "Error committing to the repository."
 
 git tag -a "$SAGE_VERSION" -m "$SAGE_VERSION_BANNER" \