Merge tag '9.7.beta5' into t/33011/remove___init___py_files_for_packa…

…ges_designated_to_be_namespace_packages

SageMath version 9.7.beta5, Release Date: 2022-07-10

.github/workflows/build.yml

@@ -32,6 +32,7 @@ jobs:
           # Install test tools.
           # Installation of python3-venv can be removed as soon as a
           # base image with a release including #33822 is available
+          apt-get update
           apt-get install -y git python3-venv
           # Reuse built SAGE_LOCAL contained in the Docker image 
           ./bootstrap

.github/workflows/ci-macos.yml

@@ -0,0 +1,226 @@
+name: CI macOS
+
+## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
+## whenever a GitHub pull request is opened or synchronized in a repository
+## where GitHub Actions are enabled.
+##
+## It builds and checks some sage spkgs as defined in TARGETS.
+##
+## A job succeeds if there is no error.
+##
+## The build is run with "make V=0", so the build logs of individual packages are suppressed.
+##
+## At the end, all package build logs that contain an error are printed out.
+##
+## After all jobs have finished (or are canceled) and a short delay,
+## tar files of all logs are made available as "build artifacts".
+
+#on: [push, pull_request]
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+    # Allow to run manually
+
+env:
+  TARGETS_PRE: all-sage-local
+  TARGETS: build doc-html
+  TARGETS_OPTIONAL: ptest
+
+jobs:
+  local-macos:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        stage: ["1", "2", "2-optional-0-o", "2-optional-p-z", "2-experimental-0-o", "2-experimental-p-z"]
+        # python3_xcode is only accepted if enough packages are available from the system
+        # --> to test "minimal", we would need https://trac.sagemath.org/ticket/30949
+        tox_env: [homebrew-macos-usrlocal-minimal, homebrew-macos-usrlocal-standard, homebrew-macos-usrlocal-maximal, homebrew-macos-usrlocal-python3_xcode-standard, conda-forge-macos-minimal, conda-forge-macos-standard, conda-forge-macos-maximal]
+        # As of 2021-12, default xcode
+        # - on macos-10.15:               12.4
+        # - on macos-latest (= macos-11): 13.1
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
+        os: [ macos-10.15, macos-latest ]
+    env:
+      TOX_ENV: local-${{ matrix.tox_env }}
+      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}--${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
+      - name: Install test prerequisites
+        run: |
+          brew install tox
+      - uses: actions/download-artifact@v2
+        with:
+          path: sage-local-artifact
+          name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        if: contains(matrix.stage, '2')
+      - name: Extract sage-local artifact
+        # This is macOS tar -- cannot use --listed-incremental
+        run: |
+          export SAGE_LOCAL=$(pwd)/.tox/$TOX_ENV/local
+          .github/workflows/extract-sage-local.sh sage-local-artifact/sage-local-*.tar
+        if: contains(matrix.stage, '2')
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          case "${{ matrix.stage }}" in
+            1)               export TARGETS_PRE="all-sage-local" TARGETS="all-sage-local" TARGETS_OPTIONAL=""
+                             ;;
+            2)               export TARGETS_PRE="all-sage-local" TARGETS="build doc-html" TARGETS_OPTIONAL="ptest"
+                             ;;
+            2-optional*)     export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
+                             targets_pattern="${{ matrix.stage }}"
+                             targets_pattern="${targets_pattern#2-optional-}"
+                             export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :optional: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
+                             ;;
+            2-experimental*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
+                             targets_pattern="${{ matrix.stage }}"
+                             targets_pattern="${targets_pattern#2-experimental-}"
+                             export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :experimental: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
+                             ;;
+          esac
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - name: Prepare sage-local artifact
+        # This also includes the copies of homebrew or conda installed in the tox environment.
+        # We use absolute pathnames in the tar file.
+        # This is macOS tar -- cannot use --remove-files.
+        # We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
+        run: |
+          mkdir -p sage-local-artifact && (cd .tox/$TOX_ENV && rm -f "local/lib64" && tar -cf - $(pwd)) > sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
+        if: contains(matrix.stage, '1')
+      - uses: actions/upload-artifact@v2
+        with:
+          path: sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
+          name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        if: always()
+
+  dist:
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 500
+      - name: fetch tags
+        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+      - name: Install bootstrap prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
+      - name: Bootstrap with sage-update-version
+        # We set SAGE_ROOT and SAGE_SRC by hand
+        # because 'sage -sh' does not work with an unconfigured tree,
+        # giving: Error: SAGE_SCRIPTS_DIR is set to a bad value
+        run: |
+          git config --global user.email "nobody@example.com"
+          git config --global user.name "Sage GitHub CI"
+          SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe) || echo "(ignoring error)"
+      - name: make dist
+        run: |
+          ./configure --enable-download-from-upstream-url && make dist
+      - uses: actions/upload-artifact@v2
+        with:
+          path: "dist/*.tar.gz"
+          name: dist
+
+  local-macos-nohomebrew:
+
+    needs: [dist]
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        os: [ macos-10.15, macos-11.0 ]
+        tox_system_factor: [macos-nobootstrap, macos-nobootstrap-python3_pythonorg]
+        tox_packages_factor: [minimal]
+        # As of 2021-03, default is 12.4
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
+        include:
+          # Test xcode 11.7 only on macos-10.15
+          - tox_system_factor: macos-nobootstrap
+            tox_packages_factor: minimal
+            xcode_version_factor: 11.7
+            os: macos-10.15
+          - tox_system_factor: macos-nobootstrap-python3_pythonorg
+            tox_packages_factor: minimal
+            xcode_version_factor: 11.7
+            os: macos-10.15
+    env:
+      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-xcode_${{ matrix.xcode_version_factor }}
+    steps:
+      - uses: actions/checkout@v2
+        if: "!contains(matrix.tox_system_factor, 'nobootstrap')"
+      - uses: actions/download-artifact@v2
+        with:
+          path: .
+          name: dist
+        if: contains(matrix.tox_system_factor, 'nobootstrap')
+      - name: Unpack sage dist
+        run: |
+          tar xf sage*.tar.gz --strip-components=1
+        if: contains(matrix.tox_system_factor, 'nobootstrap')
+      - name: Move homebrew away
+        run: |
+          (cd /usr/local && for a in bin etc include lib opt sbin share; do sudo mv $a $a-moved; done)
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
+      - name: Install test prerequisites
+        run: |
+          sudo /usr/bin/python3 -m pip install tox
+      - name: Install python3 from python.org
+        # As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
+        # Python 3.7.7 is installed on GitHub Actions runners.  But we install our own copy from the python.org binary package.
+        run: |
+          curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
+          sudo installer -verbose -pkg python3.pkg -target /
+        if: contains(matrix.tox_system_factor, 'python3_pythonorg')
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()

.github/workflows/cygwin.yml

@@ -0,0 +1,151 @@
+name: Reusable workflow for Cygwin
+
+on:
+  workflow_call:
+    inputs:
+      targets_pre:
+        default: build/make/Makefile
+        type: string
+      targets:
+        type: string
+      targets_optional:
+        default: build/make/Makefile
+        type: string
+      prefix:
+        required: false
+        default: /opt/sage
+        type: string
+      toxenv:
+        required: false
+        default: local-cygwin-choco-standard
+        type: string
+      #
+      # For use in multi-stage CIs
+      #
+      stage:
+        required: false
+        default: main
+        type: string
+      previous_stages:
+        required: false
+        type: string
+      #
+      # For use in upstream CIs
+      #
+      upstream_artifact:
+        required: false
+        type: string
+      sage_repo:
+        required: false
+        type: string
+      sage_trac_git:
+        required: false
+        type: string
+      sage_trac_ticket:
+        required: false
+        type: string
+      sage_ref:
+        required: false
+        type: string
+
+jobs:
+  stage:
+    env:
+      MAKE: make -j8
+      SAGE_NUM_THREADS: 3
+      CYGWIN: winsymlinks:native
+      EXTRA_CONFIGURE_ARGS: --enable-fat-binary
+      SAGE_LOCAL: ${{ inputs.prefix }}-${{ github.sha }}
+      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-${{ inputs.toxenv }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-${{ inputs.toxenv }}
+
+    runs-on: windows-latest
+
+    steps:
+    - run: |
+        git config --global core.autocrlf false
+        git config --global core.symlinks true
+
+    - name: install cygwin with choco
+      shell: bash {0}
+      run: |
+        choco --version
+        choco install git python39 python39-pip --source cygwin
+
+    - name: Check out SageMath
+      uses: actions/checkout@v2
+      with:
+        repository: ${{ inputs.sage_repo }}
+        ref: ${{ inputs.sage_ref }}
+        fetch-depth: 2000
+
+    - name: Check out git-trac-command
+      uses: actions/checkout@v2
+      with:
+        repository: sagemath/git-trac-command
+        path: git-trac-command
+      if: inputs.sage_trac_git != ''
+    - name: Check out SageMath from trac.sagemath.org
+      shell: bash {0}
+      # Random sleep and retry to limit the load on trac.sagemath.org
+      run: |
+        git config --global user.email "ci-sage@example.com"
+        git config --global user.name "ci-sage workflow"
+        if [ ! -d .git ]; then git init; fi; git remote add trac ${{ inputs.sage_trac_git }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch ${{ inputs.sage_trac_ticket }}; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
+      if: inputs.sage_trac_git != ''
+
+    - name: Download upstream artifact
+      uses: actions/download-artifact@v2
+      with:
+        path: upstream
+        name: ${{ inputs.upstream_artifact }}
+      if: inputs.upstream_artifact
+    - name: Update Sage packages from upstream artifact
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && git config --global --add safe.directory $(pwd) && ls -l upstream/ && export PATH="$(pwd)/build/bin:$PATH:/usr/local/bin:/usr/bin" && (cd upstream && bash -x update-pkgs.sh) && git diff'
+      if: inputs.upstream_artifact
+
+    - name: Download sage-local artifact
+      uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        path: C:\\tools\\cygwin\\tmp
+      if: inputs.previous_stages
+    - name: Extract sage-local artifact
+      run: |
+        C:\\tools\\cygwin\\bin\\dash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && .github/workflows/extract-sage-local.sh /tmp/sage-local-*.tar && tar --create --listed-incremental=/tmp/sage-local.snar --file /dev/null "${{ env.SAGE_LOCAL }}"'
+      if: inputs.previous_stages
+    - name: tox
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -x -c 'cat /proc/cpuinfo'
+        C:\\tools\\cygwin\\bin\\bash -l -x -c 'python3.9 -m pip install tox'
+        C:\\tools\\cygwin\\bin\\bash -l -x -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && PREFIX="${{ env.SAGE_LOCAL }}" TARGETS_PRE="${{ inputs.targets_pre }}" TARGETS_OPTIONAL="${{ inputs.targets_optional }}" tox -e ${{ inputs.toxenv }} -- ${{ inputs.targets }}'
+    - name: Prepare logs artifact
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in "${{ env.SAGE_LOCAL }}"/var/tmp/sage/build/*; do if [ -d "$a" ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/"$(basename "$a")".tar" "$a"; fi; done; cp -r logs/* "artifacts/$LOGS_ARTIFACT_NAME"'
+      if: always()
+    - uses: actions/upload-artifact@v2
+      with:
+        path: artifacts
+        name: ${{ env.LOGS_ARTIFACT_NAME }}
+      if: always()
+    - name: Print out logs for immediate inspection
+      # The markup in the output is a GitHub Actions logging command
+      # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
+      shell: bash
+      run: |
+        find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
+      if: always()
+    - name: Prepare sage-local artifact
+      # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
+      # We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f "${{ env.SAGE_LOCAL }}"/lib64; tar -cf /tmp/sage-local-${{ inputs.stage }}.tar --listed-incremental=/tmp/sage-local.snar "${{ env.SAGE_LOCAL }}"'
+      if: always()
+    - uses: actions/upload-artifact@v2
+      # upload-artifact@v2 does not support whitespace in file names.
+      # so we tar up the directory ourselves
+      with:
+        path: C:\\tools\\cygwin\\tmp\\sage-local-${{ inputs.stage }}.tar
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+      if: always()