extract script from .travis.yml + generally improve readability

.travis.yml

@@ -4,9 +4,9 @@ os: linux
 language: minimal
 cache:
   directories:
-  - depends/built
-  - depends/sdk-sources
-  - $HOME/.ccache
+    - depends/built
+    - depends/sdk-sources
+    - $HOME/.ccache
 git:
   depth: false  # full clone for git subtree check, this works around issue #12388
 env:
@@ -21,66 +21,114 @@ env:
     - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
     - SDK_URL=https://bitcoincore.org/depends-sources/sdks
     - WINEDEBUG=fixme-all
-  matrix:
+stages:
+  - lint
+  - test
+before_install:
+  - set -o errexit; source .travis/test_03_before_install.sh
+install:
+  - set -o errexit; source .travis/test_04_install.sh
+before_script:
+  - set -o errexit; source .travis/test_05_before_script.sh
+script:
+  - set -o errexit; source .travis/test_06_script.sh
+after_script:
+  - echo $TRAVIS_COMMIT_RANGE
+  - echo $TRAVIS_COMMIT_LOG
+jobs:
+  include:
+    - stage: lint
+      env:
+      dist: trusty
+      os: linux
+      sudo: false
+      cache: false
+      language: python
+      python: '2.7'
+      addons:
+        apt:
+          packages:
+            - python3-flake8
+      before_install:
+        - set -o errexit; source .travis/lint_03_before_install.sh
+      install:
+        - set -o errexit; source .travis/lint_04_install.sh
+      before_script:
+        - set -o errexit; source .travis/lint_05_before_script.sh
+      script:
+        - set -o errexit; source .travis/lint_06_script.sh
 # ARM
-    - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf python3-pip" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" UNITE_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
+    - stage: test
+      env: >-
+        HOST=arm-linux-gnueabihf
+        PACKAGES="g++-arm-linux-gnueabihf python3-pip"
+        DEP_OPTS="NO_QT=1"
+        CHECK_DOC=1
+        GOAL="install"
+        UNITE_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
 # Win32
-    - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6" RUN_TESTS=true GOAL="install" UNITE_CONFIG="--enable-reduce-exports"
+    - stage: test
+      env: >-
+        HOST=i686-w64-mingw32
+        DPKG_ADD_ARCH="i386"
+        DEP_OPTS="NO_QT=1"
+        PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6"
+        RUN_TESTS=true
+        GOAL="install"
+        UNITE_CONFIG="--enable-reduce-exports"
 # Qt4 & system libs
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="python3-zmq qt4-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev xvfb" NO_DEPENDS=1 NEED_XVFB=1 RUN_TESTS=true GOAL="install" UNITE_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt4 CPPFLAGS=-DDEBUG_LOCKORDER"
+    - stage: test
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="python3-zmq qt4-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev xvfb"
+        NO_DEPENDS=1
+        NEED_XVFB=1
+        RUN_TESTS=true
+        GOAL="install"
+        UNITE_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt4 CPPFLAGS=-DDEBUG_LOCKORDER"
 # 32-bit + dash
-    - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib python3-zmq" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" UNITE_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash"
+    - stage: test
+      env: >-
+        HOST=i686-pc-linux-gnu
+        PACKAGES="g++-multilib python3-zmq"
+        DEP_OPTS="NO_QT=1"
+        RUN_TESTS=true
+        GOAL="install"
+        UNITE_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
+        USE_SHELL="/bin/dash"
 # Win64
-    - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6" RUN_TESTS=true GOAL="install" UNITE_CONFIG="--enable-reduce-exports"
+    - stage: test
+      env: >-
+        HOST=x86_64-w64-mingw32
+        DPKG_ADD_ARCH="i386"
+        DEP_OPTS="NO_QT=1"
+        PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6"
+        RUN_TESTS=true
+        GOAL="install"
+        UNITE_CONFIG="--enable-reduce-exports"
 # x86_64 Linux (uses qt5 dev package instead of depends Qt to speed up build and avoid timeout)
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" RUN_TESTS=true GOAL="install" UNITE_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER"
+    - stage: test
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev"
+        DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
+        RUN_TESTS=true
+        GOAL="install"
+        UNITE_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER"
 # x86_64 Linux, No wallet
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" UNITE_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
+    - stage: test
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="python3"
+        DEP_OPTS="NO_WALLET=1"
+        RUN_TESTS=true
+        GOAL="install"
+        UNITE_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
 # Cross-Mac
-    - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" UNITE_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" OSX_SDK=10.11 GOAL="deploy"
+    - stage: test
+      env: >-
+        HOST=x86_64-apple-darwin11
+        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev"
+        UNITE_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
+        OSX_SDK=10.11 GOAL="deploy"
 
-before_install:
-    - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
-install:
-    - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
-    - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi
-    - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi
-    - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then travis_retry pip3 install flake8 --user; fi
-before_script:
-    - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/commit-script-check.sh $TRAVIS_COMMIT_RANGE; fi
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/crypto/ctaes; fi
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/secp256k1; fi
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/univalue; fi
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/leveldb; fi
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi
-    - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-rpc-mappings.py .; fi
-    - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/lint-all.sh; fi
-    - unset CC; unset CXX
-    - mkdir -p depends/SDKs depends/sdk-sources
-    - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
-    - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
-    - if [ -z "$NO_DEPENDS" ]; then make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; fi
-    # Start xvfb if needed, as documented at https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
-    - if [ "$NEED_XVFB" = 1 ]; then export DISPLAY=:99.0; /sbin/start-stop-daemon --start --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac; fi
-script:
-    - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "unite/unite" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi
-    - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "unite/unite" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then contrib/verify-commits/verify-commits.sh; fi
-    - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
-    - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi
-    - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
-    - UNITE_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
-    - if [ -z "$NO_DEPENDS" ]; then depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE; fi
-    - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
-    - mkdir build && cd build
-    - ../configure --cache-file=config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
-    - make distdir VERSION=$HOST
-    - cd unite-$HOST
-    - ./configure --cache-file=../config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
-    - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
-    - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
-    - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi
-    - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude feature_pruning,feature_dbcrash"; fi
-    - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet ${extended}; fi
-after_script:
-    - echo $TRAVIS_COMMIT_RANGE
-    - echo $TRAVIS_COMMIT_LOG

.travis/README.md

@@ -0,0 +1,8 @@
+## travis build scripts
+
+The `.travis` directory contains scripts for each build step in each build stage.
+Currently the travis build defines two stages `lint` and `test`. Each stage has
+it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle).
+Every script in here is named and numbered according to which stage and lifecycle
+step it belongs to.
+

.travis/lint_06_script.sh

@@ -0,0 +1,10 @@
+
+
+./contrib/devtools/git-subtree-check.sh src/crypto/ctaes
+./contrib/devtools/git-subtree-check.sh src/secp256k1
+./contrib/devtools/git-subtree-check.sh src/univalue
+./contrib/devtools/git-subtree-check.sh src/leveldb
+./contrib/devtools/check-doc.py
+./contrib/devtools/check-rpc-mappings.py .
+./contrib/devtools/lint-all.sh
+

.travis/test_03_before_install.sh

@@ -0,0 +1,5 @@
+
+
+PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
+export PATH
+

.travis/test_04_install.sh

@@ -0,0 +1,13 @@
+
+
+if [ -n "$DPKG_ADD_ARCH" ]; then
+  sudo dpkg --add-architecture "$DPKG_ADD_ARCH"
+fi
+
+if [ -n "$PACKAGES" ]; then
+  travis_retry sudo apt-get update
+fi
+
+if [ -n "$PACKAGES" ]; then
+  travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES
+fi

.travis/test_05_before_script.sh

@@ -0,0 +1,24 @@
+
+
+unset CC
+unset CXX
+
+mkdir -p depends/SDKs depends/sdk-sources
+
+if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+fi
+
+if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+fi
+
+if [ -z "$NO_DEPENDS" ]; then
+  make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+fi
+
+# Start xvfb if needed, as documented at https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
+if [ "$NEED_XVFB" = 1 ]; then
+  export DISPLAY=:99.0; /sbin/start-stop-daemon --start --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac
+fi
+

.travis/test_06_script.sh

@@ -0,0 +1,45 @@
+
+
+TRAVIS_COMMIT_LOG=$(git log --format=fuller -1)
+export TRAVIS_COMMIT_LOG
+
+if [ -n "$USE_SHELL" ]; then
+  export CONFIG_SHELL="$USE_SHELL"
+fi
+
+OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
+UNITE_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
+
+if [ -z "$NO_DEPENDS" ]; then
+  depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE
+fi
+
+test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
+
+mkdir build && cd build
+
+../configure --cache-file=config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
+
+make distdir VERSION=$HOST
+
+cd unite-$HOST
+
+./configure --cache-file=../config.cache $UNITE_CONFIG_ALL $UNITE_CONFIG || ( cat config.log && false)
+
+make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
+
+export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
+
+if [ "$RUN_TESTS" = "true" ]; then
+  travis_wait 30 make $MAKEJOBS check VERBOSE=1
+fi
+
+if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
+  extended="--extended --exclude feature_pruning,feature_dbcrash"
+fi
+
+if [ "$RUN_TESTS" = "true" ]; then
+  test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet ${extended}
+fi
+
+