Skip to content
This repository has been archived by the owner on Jan 30, 2023. It is now read-only.

Commit

Permalink
Merge branch 't/29950/build_sagelib_using_installed_sage_setup' into …
Browse files Browse the repository at this point in the history
…t/30010/split_sage_setup_docbuild_out_to_a_separate_package
  • Loading branch information
Matthias Koeppe committed Aug 10, 2020
2 parents 39781d1 + ccc67b0 commit 0237faa
Show file tree
Hide file tree
Showing 371 changed files with 13,051 additions and 5,557 deletions.
4 changes: 4 additions & 0 deletions .ci/push-dockerhub.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,8 @@ if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then
else
cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin
docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG

# For historical reasons, we also provide a -py3 tag. It's identical to the non-py3 tag.
docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
fi
144 changes: 144 additions & 0 deletions .github/workflows/tox-experimental.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
name: Test experimental packages with tox

## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
## whenever a GitHub pull request is opened or synchronized in a repository
## where GitHub Actions are enabled.
##
## It builds and checks some sage spkgs as defined in TARGETS.
##
## A job succeeds if there is no error.
##
## The build is run with "make V=0", so the build logs of individual packages are suppressed.
##
## At the end, all package build logs that contain an error are printed out.
##
## After all jobs have finished (or are canceled) and a short delay,
## tar files of all logs are made available as "build artifacts".

#on: [push, pull_request]

on:
pull_request:
types: [opened, synchronize]
push:
tags:
- '*'

env:
TARGETS_PRE: build/make/Makefile
TARGETS: build/make/Makefile
# TARGETS_OPTIONAL see below

jobs:
docker:
runs-on: ubuntu-latest
strategy:
fail-fast: false
max-parallel: 6
matrix:
tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386]
tox_packages_factor: [maximal]
targets_pattern: [0-g, h-o, p, q-z]
env:
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 500
- name: fetch tags
run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
- name: free disk space
run: |
sudo swapoff -a
sudo rm -f /swapfile
sudo apt clean
docker rmi $(docker image ls -aq)
df -h
- name: Install test prerequisites
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
- run: |
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
- name: Copy logs from the docker image or build container
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
if: always()
- uses: actions/upload-artifact@v1
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- name: Push docker images
run: |
if [ -f .tox/$TOX_ENV/Dockertags ]; then
TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
if [ -z "$TOKEN" ]; then
TOKEN="${{ secrets.GITHUB_TOKEN }}"
fi
echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
for a in $(cat .tox/$TOX_ENV/Dockertags); do
FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a
docker tag $a $FULL_TAG
echo Pushing $FULL_TAG
docker push $FULL_TAG
done || echo "(Ignoring errors)"
fi
if: always()

local-macos:

runs-on: macos-latest
strategy:
fail-fast: false
max-parallel: 3
matrix:
tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
tox_packages_factor: [maximal]
targets_pattern: [0-g, h-o, p, q-z]
env:
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
steps:
- uses: actions/checkout@v2
- name: Install test prerequisites
run: |
brew install tox
- name: Install python3 from python.org
# As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
# Python 3.7.7 is installed on GitHub Actions runners. But we install our own copy from the python.org binary package.
run: |
curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
sudo installer -verbose -pkg python3.pkg -target /
if: contains(matrix.tox_system_factor, 'python3_pythonorg')
- name: Build and test with tox
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
# For doctesting, we use a lower parallelization to avoid timeouts.
run: |
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
- name: Prepare logs artifact
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v1
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
12 changes: 10 additions & 2 deletions .github/workflows/tox-optional.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Run SAGE_ROOT/tox.ini TARGETS_OPTIONAL
name: Test optional packages with tox

## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
## whenever a GitHub pull request is opened or synchronized in a repository
Expand Down Expand Up @@ -27,7 +27,7 @@ on:
env:
TARGETS_PRE: build/make/Makefile
TARGETS: build/make/Makefile
TARGETS_OPTIONAL: 4ti2 pynormaliz qepcad lrslib latte_int topcom barvinok isl qhull sage_numerical_backends_coin primecount plantri polymake jupymake kenzo libsemigroups mcqd meataxe mpfrcx openssl p_group_cohomology rst2ipynb sirocco tdlib tides
# TARGETS_OPTIONAL see below

jobs:
docker:
Expand All @@ -38,10 +38,14 @@ jobs:
matrix:
tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386]
tox_packages_factor: [maximal]
targets_pattern: [0-g, h-o, p, q-z]
env:
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
# Test all optional packages, but do not test huge packages,
# and do not test packages that require external software
TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :optional: | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
steps:
- uses: actions/checkout@v2
with:
Expand Down Expand Up @@ -104,9 +108,13 @@ jobs:
matrix:
tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
tox_packages_factor: [maximal]
targets_pattern: [0-g, h-o, p, q-z]
env:
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
# Test all optional packages, but do not test huge packages
# and do not test packages that require external software
TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :optional: | grep -v database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
steps:
- uses: actions/checkout@v2
- name: Install test prerequisites
Expand Down
88 changes: 88 additions & 0 deletions .github/workflows/tox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,3 +176,91 @@ jobs:
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()

dist:

runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 500
- name: fetch tags
run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
- name: Install bootstrap prerequisites
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install $(sed "s/#.*//;" build/pkgs/debian-bootstrap.txt)
- name: Bootstrap with sage-update-version
# We set SAGE_ROOT and SAGE_SRC by hand
# because 'sage -sh' does not work with an unconfigured tree,
# giving: Error: SAGE_SCRIPTS_DIR is set to a bad value
run: |
git config --global user.email "nobody@example.com"
git config --global user.name "Sage GitHub CI"
SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe)
- name: make dist
run: |
./configure && make dist
- uses: actions/upload-artifact@v2
with:
path: "dist/*.tar.gz"
name: dist

local-macos-nohomebrew:

needs: [dist]

runs-on: macos-latest
strategy:
fail-fast: false
max-parallel: 4
matrix:
tox_system_factor: [macos-nobootstrap, macos-nobootstrap-python3_pythonorg]
tox_packages_factor: [minimal]
env:
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
steps:
- uses: actions/checkout@v2
if: "!contains(matrix.tox_system_factor, 'nobootstrap')"
- uses: actions/download-artifact@v2
with:
path: .
name: dist
if: contains(matrix.tox_system_factor, 'nobootstrap')
- name: Unpack sage dist
run: |
tar xf sage*.tar.gz --strip-components=1
if: contains(matrix.tox_system_factor, 'nobootstrap')
- name: Move homebrew away
run: |
(cd /usr/local && for a in bin etc include lib opt sbin share; do mv $a $a-moved; done)
- name: Install test prerequisites
run: |
/usr/bin/python3 -m pip install --user tox
- name: Install python3 from python.org
# As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
# Python 3.7.7 is installed on GitHub Actions runners. But we install our own copy from the python.org binary package.
run: |
curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
sudo installer -verbose -pkg python3.pkg -target /
if: contains(matrix.tox_system_factor, 'python3_pythonorg')
- name: Build and test with tox
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
# For doctesting, we use a lower parallelization to avoid timeouts.
run: |
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
- name: Prepare logs artifact
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v1
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,4 @@ gitlab-build-docker.log

/.tox
/prefix
worktree*
40 changes: 0 additions & 40 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,43 +193,3 @@ push-dockerhub-dev:
script:
- . .ci/pull-gitlab.sh sagemath-dev
- sh .ci/push-dockerhub.sh sagemath-dev

.py3:
variables:
WITH_PYTHON: 3
DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop-py3

build-from-latest-py3:
extends:
- build-from-latest
- .py3

build-from-clean-py3:
extends:
- build-from-clean
- .py3

test-dev-py3:
extends:
- test-dev
- .py3

test-cli-py3:
extends:
- test-cli
- .py3

test-jupyter-py3:
extends:
- test-jupyter
- .py3

push-dockerhub-py3:
extends:
- push-dockerhub
- .py3

push-dockerhub-dev-py3:
extends:
- push-dockerhub-dev
- .py3
7 changes: 6 additions & 1 deletion .homebrew-build-env
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,19 @@ export PKG_CONFIG_PATH
LIBRARY_PATH="$HOMEBREW/lib$LIBRARY_PATH"
[ -z "$CPATH" ] || CPATH=":${CPATH}"
CPATH="$HOMEBREW/include$CPATH"
for l in readline; do
for l in readline ; do
if [ -d "$HOMEBREW/opt/$l/lib" ]; then
LIBRARY_PATH="$HOMEBREW/opt/$l/lib:$LIBRARY_PATH"
fi
if [ -d "$HOMEBREW/opt/$l/include" ]; then
CPATH="$HOMEBREW/opt/$l/include:$CPATH"
fi
done
for l in "gcc@9/lib/gcc/9"; do
if [ -d "$HOMEBREW/opt/$l" ]; then
LIBRARY_PATH="$HOMEBREW/opt/$l:$LIBRARY_PATH"
fi
done
export LIBRARY_PATH
export CPATH
for l in gettext; do
Expand Down
1 change: 0 additions & 1 deletion COPYING.txt
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@ sage_scripts GPLv2+
sagenb GPLv3
sagetex GPLv2+ (CC BY-SA 3.0 for included documentation)
scipy Modified BSD
scons MIT License
setuptools Python License
singular GPLv2 or GPLv3 (see below)
six MIT License
Expand Down
5 changes: 0 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -255,11 +255,6 @@ Guide](https://doc.sagemath.org/html/en/installation).
powerful machines, you might even consider `-j16`, as building with
more jobs than CPU cores can speed things up further.

If you want to run the test suite for each individual Sage package
as it gets installed, type `export SAGE_CHECK="yes"`. This will run
each test suite, raising an error if any failure occurs. If set to
``warn``, then only a warning is printed in this case.

To reduce the terminal output during the build, type `export V=0`.
(`V` stands for "verbosity".)

Expand Down
2 changes: 1 addition & 1 deletion VERSION.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SageMath version 9.2.beta5, Release Date: 2020-07-12
SageMath version 9.2.beta7, Release Date: 2020-08-02
Loading

0 comments on commit 0237faa

Please sign in to comment.