Merge branch 'develop' into develop

oliverdaisey · Jan 9, 2024 · 1ab74b4 · 1ab74b4
2 parents ed9087d + e249bef
commit 1ab74b4
Show file tree

Hide file tree

Showing 5,086 changed files with 124,139 additions and 96,409 deletions.
diff --git a/.ci/create-changes-html.sh b/.ci/create-changes-html.sh
@@ -0,0 +1,97 @@
+#!/bin/sh
+if [ $# != 2 ]; then
+    echo >&2 "usage: $0 BASE_DOC_COMMIT DOC_REPO"
+    echo >&2 "creates CHANGES.html in the current directory"
+    echo >&2 "for the diffs of DOC_REPO against BASE_DOC_COMMIT"
+    exit 1
+fi
+BASE_DOC_COMMIT="$1"
+DOC_REPOSITORY="$2"
+
+# Wipe out chronic diffs between old doc and new doc
+(cd $DOC_REPOSITORY && find . -name "*.html" | xargs sed -i -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
+# Create CHANGES.html
+echo '<html>' > CHANGES.html
+echo '<head>' >> CHANGES.html
+echo '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">' >> CHANGES.html
+echo '<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>' >> CHANGES.html
+echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
+cat >> CHANGES.html << EOF
+<script>
+document.addEventListener('DOMContentLoaded', () => {
+const diffSite = 'https://pianomister.github.io/diffsite'
+const baseDocURL = 'https://sagemath-tobias.netlify.app'
+const diffParagraphs = document.querySelectorAll('p.diff');
+diffParagraphs.forEach(paragraph => {
+  const rootURL = window.location.origin;
+  const docAnchor = paragraph.querySelector('a');  // first "a" element
+  const url = new URL(docAnchor.href);
+  const path = url.pathname;
+  const anchor = document.createElement('a');
+  anchor.href = diffSite + '/?url1=' + rootURL + path + '&url2=' + baseDocURL + path;
+  anchor.textContent = 'compare with the base';
+  anchor.setAttribute('target', '_blank');
+  paragraph.appendChild(anchor);
+  paragraph.innerHTML += '&nbsp;';
+  const hunkAnchors = paragraph.querySelectorAll('a.hunk');
+  hunkAnchors.forEach(hunkAnchor => {
+    const url = new URL(hunkAnchor.href);
+    const path = url.pathname;
+    const pathHash = path + url.hash.replace('#', '%23');
+    const anchor = document.createElement('a');
+    anchor.href = diffSite + '/?url1=' + rootURL + pathHash + '&url2=' + baseDocURL + path;
+    anchor.textContent = hunkAnchor.textContent;
+    anchor.setAttribute('target', '_blank');
+    paragraph.appendChild(anchor);
+    paragraph.innerHTML += '&nbsp;';
+  });
+});
+});
+</script>
+EOF
+echo '</head>' >> CHANGES.html
+echo '<body>' >> CHANGES.html
+(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- *.html) > diff.txt
+python3 - << EOF
+import os, re, html
+with open('diff.txt', 'r') as f:
+    diff_text = f.read()
+diff_blocks = re.split(r'^(?=diff --git)', diff_text, flags=re.MULTILINE)
+out_blocks = []
+for block in diff_blocks:
+    match = re.search(r'^diff --git a/(.*) b/\1', block, flags=re.MULTILINE)
+    if match:
+        doc = match.group(1)
+        file_path = os.path.join('$DOC_REPOSITORY', doc)
+        try:
+            with open(file_path, 'r') as file:
+                content = file.readlines()
+        except FileNotFoundError:
+            content = []
+        count = 0
+        for line in block.splitlines():
+            if line.startswith('@@ -'):
+                search_result = re.search(r'@@ -(\d+),(\d+) \+(\d+),(\d+)', line)
+                if search_result:
+                    line_number = int(search_result.group(3))
+                    for i in range(line_number - 1, -1, -1):
+                        if content[i].startswith('<'):
+                            count += 1
+                            content[i] = f'<span id="hunk{count}" style="visibility: hidden;"></span>' + content[i]
+                            break
+        if content:
+            with open(file_path, 'w') as file:
+                file.writelines(content)
+        path = 'html/' + doc
+        hunks = '&nbsp;'.join(f'<a href="{path}#hunk{i+1}" class="hunk" target="_blank">#{i + 1}</a>' for i in range(count))
+        out_blocks.append(f'<p class="diff"><a href="{path}">{doc}</a>&nbsp;' + hunks + '&emsp;</p>'
+                            + '\n<pre><code class="language-diff">'
+                            + html.escape(block).strip() + '</code></pre>')
+output_text = '\n'.join(out_blocks)
+with open('diff.html', 'w') as f:
+    f.write(output_text)
+EOF
+cat diff.html >> CHANGES.html
+echo '</body>' >> CHANGES.html
+echo '</html>' >> CHANGES.html
+rm diff.txt diff.html
diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh
diff --git a/.ci/merge-fixes.sh b/.ci/merge-fixes.sh
@@ -0,0 +1,82 @@
+#!/bin/sh
+# Apply open PRs labeled "blocker" from sagemath/sage as patches.
+# This script is invoked by various workflows in .github/workflows
+#
+# The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
+# to customize this for CI runs in forks:
+#
+# - If set to a whitespace-separated list of repositories, use them instead of sagemath/sage.
+# - If set to "none", do not apply any PRs.
+#
+# https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-a-repository
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="ci-sage@example.com"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+mkdir -p upstream
+for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
+    case $REPO in
+        none)
+            echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
+            ;;
+        */*)
+            echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22"
+            GH="gh -R $REPO"
+            REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
+            PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)"
+            date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date  # Record the date, for future reference
+            if [ -z "$PRs" ]; then
+                echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO."
+            else
+                echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)"
+                git tag -f test_base
+                git commit -q -m "Uncommitted changes" --no-allow-empty -a
+                for a in $PRs; do
+                    # We used to pull the branch and merge it (after unshallowing), but when run on PRs that are
+                    # based on older releases, it has the side effect of updating to this release,
+                    # which may be confusing.
+                    #
+                    # Here, we obtain the "git am"-able patch of the PR branch.
+                    # This also makes it unnecessary to unshallow the repository.
+                    #
+                    # Considered alternative: Use https://github.com/$REPO/pull/$a.diff,
+                    # which squashes everything into one diff without commit metadata.
+                    PULL_URL="https://github.com/$REPO/pull/$a"
+                    PULL_FILE="$REPO_FILE-$a"
+                    PATH=build/bin:$PATH build/bin/sage-download-file --quiet "$PULL_URL.patch" $PULL_FILE.patch
+                    date -u +"%Y-%m-%dT%H:%M:%SZ" > $PULL_FILE.date  # Record the date, for future reference
+                    LAST_SHA=$(sed -n -E '/^From [0-9a-f]{40}/s/^From ([0-9a-f]{40}).*/\1/p' $PULL_FILE.patch | tail -n 1)
+                    COMMITS_URL="https://github.com/$REPO/commits/$LAST_SHA"
+                    echo "::group::Applying PR $PULL_URL @ $COMMITS_URL as a patch"
+                    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME applying $PULL_URL @ $COMMITS_URL"
+                    if git am --signoff --empty=keep < $PULL_FILE.patch; then
+                        echo "---- Applied patch --------------------------------------------------------------------------------"
+                        cat $PULL_FILE.patch
+                        echo "--------------------------------------------------------------------8<-----------------------------"
+                        echo "::endgroup::"
+                    elif git am --abort \
+                            && if git fetch --unshallow --all > /dev/null 2>&1; then echo "Unshallowed"; fi \
+                            && echo "Retrying with 3-way merge" \
+                            && git am --empty=keep --3way < $PULL_FILE.patch; then
+                        echo "---- Applied patch --------------------------------------------------------------------------------"
+                        cat $PULL_FILE.patch
+                        echo "--------------------------------------------------------------------8<-----------------------------"
+                        echo "::endgroup::"
+                    else
+                        echo "---- Failure applying patch -----------------------------------------------------------------------"
+                        git am --signoff --show-current-patch=diff
+                        echo "--------------------------------------------------------------------8<-----------------------------"
+                        echo "::endgroup::"
+                        echo "Failure applying $PULL_URL as a patch, resetting"
+                        git am --signoff --abort
+                    fi
+                done
+            fi
+            ;;
+        *)
+            echo "Repository variable SAGE_CI_FIXES_FROM_REPOSITORIES, if set, should be a whitespace-separated list of repositories or 'none'"
+            echo "Got: $SAGE_CI_FIXES_FROM_REPOSITORIES"
+            exit 1
+            ;;
+    esac
+done
diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh
diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+if [ $# != 2 ]; then
+    echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY"
+    echo >&2 "Ensures that the current working directory is a git repository,"
+    echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME."
+fi
+WORKTREE_NAME="$1"
+WORKTREE_DIRECTORY="$2"
+
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="ci-sage@example.com"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+
+# Set globally for other parts of the workflow
+git config --global user.name "$GIT_AUTHOR_NAME"
+git config --global user.email "$GIT_AUTHOR_EMAIL"
+
+set -ex
+
+# If actions/checkout downloaded our source tree using the GitHub REST API
+# instead of with git (because do not have git installed in our image),
+# we first make the source tree a repo.
+if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+
+# Tag this state of the source tree "new". This is what we want to build and test.
+git tag -f new
+
+# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
+# But $WORKTREE_DIRECTORY is not a git repository.
+# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
+# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
+# Then we update worktree and index with "git reset --hard new".
+# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
+# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
+# The changed files now show up as uncommitted changes.
+# The final "git add -N" makes sure that files that were added in "new" do not show
+# as untracked files, which would be removed by "git clean -fx".
+git worktree add --detach $WORKTREE_NAME
+rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
+rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
+if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
diff --git a/.devcontainer/onCreate-conda.sh b/.devcontainer/onCreate-conda.sh
@@ -4,7 +4,7 @@ set -e
 # Create conda environment
 ./bootstrap-conda
 conda install mamba -n base -c conda-forge -y
-mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml
+mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml
 conda init bash
 
 # Build sage