From 3332a899abd86dbe03b9c7834eb898cb5d4f86ff Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 10:39:48 -0700 Subject: [PATCH 1/8] build/sage_bootstrap/download/mirror_list.py: Remove sagepad.org --- build/sage_bootstrap/download/mirror_list.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index 12868bf6084..3a4b9d22fc5 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -200,8 +200,6 @@ def __iter__(self): pass for mirror in self.mirrors: yield mirror - # If all else fails: Try the packages we host ourselves - yield 'http://sagepad.org/' @property def fastest(self): From dd939ff74e271282a217ce5bcbeb8122672bdc0d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 12:25:44 -0700 Subject: [PATCH 2/8] sage_bootstrap.{tarball, download.mirror_list}: Lower layout of the mirror to mirror_list --- build/sage_bootstrap/download/mirror_list.py | 4 +++- build/sage_bootstrap/tarball.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index 3a4b9d22fc5..62f622b1392 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -199,7 +199,9 @@ def __iter__(self): except KeyError: pass for mirror in self.mirrors: - yield mirror + if not mirror.endswith('/'): + mirror += '/' + yield mirror + '/'.join(['spkg', 'upstream', '${SPKG}']) @property def fastest(self): diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index 18e3da97af8..e868c5fa51c 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -159,7 +159,10 @@ def download(self, allow_upstream=False): successful_download = False log.info('Attempting to download package {0} from mirrors'.format(self.filename)) for mirror in MirrorList(): - url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) + url = mirror.replace('${SPKG}', self.package.name) + if not url.endswith('/'): + url += '/' + url += self.filename log.info(url) try: Download(url, destination).run() From 2dfe081632fd4ba254cc6d3e34647f198151ab18 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 12:28:18 -0700 Subject: [PATCH 3/8] sage_bootstrap.download.mirror_list: Delay downloading/reading/ranking to first use --- build/sage_bootstrap/download/mirror_list.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index 62f622b1392..f03d03aabb4 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -51,7 +51,12 @@ class MirrorList(object): def __init__(self): self.filename = MIRRORLIST_FILENAME - self.mirrors = None + self._mirrors = None + + @property + def mirrors(self): + if self._mirrors is not None: + return self._mirrors try: self.mirrorfile = open(self.filename, 'r+t') @@ -67,8 +72,10 @@ def __init__(self): # process while we waited for the lock? Check again. if self._must_refresh(): self._refresh() - if self.mirrors is None: - self.mirrors = self._load() + if self._mirrors is None: + self._mirrors = self._load() + + return self._mirrors def _load(self, mirror_list=None): """ @@ -147,7 +154,7 @@ def _rank_mirrors(self): log.info('Cannot time mirrors via proxy, using default order') else: timed_mirrors.sort() - self.mirrors = [m[1] for m in timed_mirrors] + self._mirrors = [m[1] for m in timed_mirrors] log.info('Fastest mirror: ' + self.fastest) def _age(self): @@ -176,12 +183,12 @@ def _refresh(self): """ log.info('Downloading the Sage mirror list') try: - with contextlib.closing(urllib.urlopen(self.URL)) as f: + with contextlib.closing(urllib.urlopen(self.url)) as f: mirror_list = f.read().decode("ascii") except IOError: log.critical('Downloading the mirror list failed, using cached version') else: - self.mirrors = self._load(mirror_list) + self._mirrors = self._load(mirror_list) self._rank_mirrors() self._save() From a37f4fdd8261e1871f446593d62cb51903a5dae8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 12:32:03 -0700 Subject: [PATCH 4/8] sage_bootstrap.download.mirror_list: Use master list of download sources in SAGE_ROOT/.upstream.d --- .upstream.d/10-SAGE_SERVER | 2 + .../20-github.com-sagemath-sage-releases | 2 + .upstream.d/30-www.sagemath.org-mirror_list | 1 + build/sage_bootstrap/download/mirror_list.py | 56 +++++++++++++++++-- 4 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 .upstream.d/10-SAGE_SERVER create mode 100644 .upstream.d/20-github.com-sagemath-sage-releases create mode 100644 .upstream.d/30-www.sagemath.org-mirror_list diff --git a/.upstream.d/10-SAGE_SERVER b/.upstream.d/10-SAGE_SERVER new file mode 100644 index 00000000000..40ee9e19450 --- /dev/null +++ b/.upstream.d/10-SAGE_SERVER @@ -0,0 +1,2 @@ +# When SAGE_SERVER is set, it should be an https/https server in the format of Sage mirrors. +${SAGE_SERVER}/spkg/upstream/${SPKG}/ diff --git a/.upstream.d/20-github.com-sagemath-sage-releases b/.upstream.d/20-github.com-sagemath-sage-releases new file mode 100644 index 00000000000..a9986bf3cce --- /dev/null +++ b/.upstream.d/20-github.com-sagemath-sage-releases @@ -0,0 +1,2 @@ +# Upstream packages as uploaded as GitHub release assets. +https://github.com/sagemath/sage/releases/download/10.1/ diff --git a/.upstream.d/30-www.sagemath.org-mirror_list b/.upstream.d/30-www.sagemath.org-mirror_list new file mode 100644 index 00000000000..951c1df6cfa --- /dev/null +++ b/.upstream.d/30-www.sagemath.org-mirror_list @@ -0,0 +1 @@ +https://www.sagemath.org/mirror_list diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index f03d03aabb4..f18c7b3e4b2 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -19,7 +19,7 @@ log = logging.getLogger() from sage_bootstrap.compat import urllib, urlparse -from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.env import SAGE_DISTFILES, SAGE_ROOT from fcntl import flock, LOCK_SH, LOCK_EX from errno import ENOLCK @@ -41,16 +41,60 @@ class MirrorListException(RuntimeError): pass -MIRRORLIST_FILENAME = os.path.join(SAGE_DISTFILES, 'mirror_list') +class MirrorList(object): + def __init__(self): + self.sources = [] + upstream_d = os.path.join(SAGE_ROOT, '.upstream.d') + for fname in sorted(os.listdir(upstream_d)): + if '~' in fname or '#' in fname: + # Ignore auto-save and backup files + continue + try: + with open(os.path.join(upstream_d, fname), 'r') as f: + for line in f: + line = line.replace('${SAGE_ROOT}', SAGE_ROOT) + line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES) + if '${SAGE_SERVER}' in line: + SAGE_SERVER = os.environ.get("SAGE_SERVER", "") + if not SAGE_SERVER: + continue + line = line.replace('${SAGE_SERVER}',) + line = line.strip() + if line.startswith('#'): + continue + if not line: + continue + if line.endswith('mirror_list'): + cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2]) + self.sources.append(MirrorList_from_url(line, cache_filename)) + else: + self.sources.append([line]) + except IOError: + # Silently ignore files that do not exist + pass -class MirrorList(object): + def __iter__(self): + """ + Iterate through the list of mirrors. + + This is the main entry point into the mirror list. Every + script should just use this function to try mirrors in order + of preference. This will not just yield the official mirrors, + but also urls for packages that are currently being tested. + """ + for source in self.sources: + for mirror in source: + yield mirror + + +class MirrorList_from_url(object): - URL = 'http://www.sagemath.org/mirror_list' MAXAGE = 24*60*60 # seconds - def __init__(self): - self.filename = MIRRORLIST_FILENAME + def __init__(self, url, filename): + self.url = url + self.filename = filename self._mirrors = None @property From 2323480b507a760d568727b11935949c52ed2d00 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 13:09:38 -0700 Subject: [PATCH 5/8] src/bin/sage-update-version: Update .upstream.d/20-github.com-sagemath-sage-releases --- .../20-github.com-sagemath-sage-releases | 1 + src/bin/sage-update-version | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.upstream.d/20-github.com-sagemath-sage-releases b/.upstream.d/20-github.com-sagemath-sage-releases index a9986bf3cce..6bccdd7ff0a 100644 --- a/.upstream.d/20-github.com-sagemath-sage-releases +++ b/.upstream.d/20-github.com-sagemath-sage-releases @@ -1,2 +1,3 @@ # Upstream packages as uploaded as GitHub release assets. +# This file is automatically updated by the sage-update-version script. https://github.com/sagemath/sage/releases/download/10.1/ diff --git a/src/bin/sage-update-version b/src/bin/sage-update-version index 29d8c794375..5ace38182ba 100755 --- a/src/bin/sage-update-version +++ b/src/bin/sage-update-version @@ -87,6 +87,25 @@ EOF # Create a top-level VERSION.txt file, which some external utilities rely on echo "$SAGE_VERSION_BANNER" > "$SAGE_ROOT/VERSION.txt" +# Add version to the front of GitHub release assets URLs. +SAGE_MINOR_VERSION=${SAGE_VERSION//.alpha*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.beta*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.dev*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.post*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.rc*/} +( echo "https://github.com/sagemath/sage/releases/download/$SAGE_MINOR_VERSION/" + sed '/^#/d' "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" +) | uniq | head -n 3 > "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases.tmp" +( cat < "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" +rm -f "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases.tmp" + +exit + # Regenerate auto-generated files tarball "$SAGE_ROOT/bootstrap" -s From 7e6526068c86068d840651c97593ddd445f38336 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 14:53:41 -0700 Subject: [PATCH 6/8] sage_bootstrap.download.mirror_list: Skip empty lines earlier --- build/sage_bootstrap/download/mirror_list.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index f18c7b3e4b2..a8baa66da2d 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -53,18 +53,18 @@ def __init__(self): try: with open(os.path.join(upstream_d, fname), 'r') as f: for line in f: + line = line.strip() + if line.startswith('#'): + continue + if not line: + continue line = line.replace('${SAGE_ROOT}', SAGE_ROOT) line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES) if '${SAGE_SERVER}' in line: SAGE_SERVER = os.environ.get("SAGE_SERVER", "") if not SAGE_SERVER: continue - line = line.replace('${SAGE_SERVER}',) - line = line.strip() - if line.startswith('#'): - continue - if not line: - continue + line = line.replace('${SAGE_SERVER}', SAGE_SERVER) if line.endswith('mirror_list'): cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2]) self.sources.append(MirrorList_from_url(line, cache_filename)) From 371be50a9df9e2ed693a6c4fafaace67f9308e3b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 14:55:32 -0700 Subject: [PATCH 7/8] src/bin/sage-update-version: Also commit --- src/bin/sage-update-version | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bin/sage-update-version b/src/bin/sage-update-version index 5ace38182ba..3b7358d2929 100755 --- a/src/bin/sage-update-version +++ b/src/bin/sage-update-version @@ -104,8 +104,6 @@ EOF ) > "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" rm -f "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases.tmp" -exit - # Regenerate auto-generated files tarball "$SAGE_ROOT/bootstrap" -s @@ -125,6 +123,7 @@ git commit -m "Updated SageMath version to $SAGE_VERSION" -- \ "$SAGE_ROOT/build/pkgs/configure/package-version.txt" \ "$SAGE_ROOT/build/pkgs/*/install-requires.txt" \ "$SAGE_ROOT"/pkgs/*/VERSION.txt \ + "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" \ || die "Error committing to the repository." git tag -a "$SAGE_VERSION" -m "$SAGE_VERSION_BANNER" \ From 423e48a3f54de03df73afe1d7fa33e8394af8d3a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 15:28:13 -0700 Subject: [PATCH 8/8] build/bin/write-dockerfile.sh: ADD .upstream.d --- build/bin/write-dockerfile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 530c2c6ec63..5e5eb522de5 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -223,6 +223,7 @@ $ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg. $ADD m4 ./m4 $ADD pkgs pkgs $ADD build ./build +$ADD .upstream.d ./.upstream.d ARG BOOTSTRAP=./bootstrap $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN