Skip to content

VTK-based merge tree backend switch #941

VTK-based merge tree backend switch

VTK-based merge tree backend switch #941

Workflow file for this run

name: test_build
on:
push:
# triggered on commit pushes on the "dev" branch
branches:
- 'dev'
# triggered on tag pushes with tags beginning with either "v" or "dev"
tags:
- 'v*'
- 'dev*'
- 'ci*'
- 'build*'
- 'test*'
pull_request:
# also triggered by pull requests on the "dev" branch
branches:
- 'dev'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
PV_TAG: v5.11.1-headless
PV_REPO: topology-tool-kit/ttk-paraview
jobs:
# ------------------#
# Test Ubuntu build #
# ------------------#
test-build-ubuntu:
runs-on: ${{ matrix.os }}
# trigger job when push event on a branch (not on a tag) only for
# the "topology-tool-kit/ttk" repository
# (pull-requests and tag pushes not affected)
if: ${{ github.repository_owner == 'topology-tool-kit' || !contains(github.ref, 'heads') }}
strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04]
env:
CCACHE_DIR: /home/runner/.ccache
CCACHE_MAXSIZE: 500M
steps:
- uses: actions/checkout@v3
name: Checkout TTK source code
- name: Install Ubuntu dependencies
run: |
sudo apt update
# TTK dependencies
sudo apt install -y \
ccache \
libboost-system-dev \
libeigen3-dev \
libgraphviz-dev \
libosmesa-dev \
libopenmpi-dev \
libsqlite3-dev \
libwebsocketpp-dev \
graphviz \
ninja-build \
zlib1g-dev \
dpkg-dev
sudo python3 -m pip install scikit-learn
- name: Install optional dependencies
uses: ./.github/actions/install-deps-unix
- name: Fetch archived ccache
continue-on-error: true
run: |
wget -O ttk-ccache.tar.gz \
https://github.com/topology-tool-kit/ttk/releases/download/ccache/ttk-ccache-${{ matrix.os }}.tar.gz
- name: Decompress ccache archive
continue-on-error: true
run: |
tar xzf ttk-ccache.tar.gz
mv .ccache /home/runner/
- name: Fetch TTK-ParaView headless Debian package
run: |
wget -O ttk-paraview-headless.deb \
https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless-${{ matrix.os }}.deb
- name: Install ParaView .deb
run: |
sudo apt install ./ttk-paraview-headless.deb
- name: Create & configure TTK build directory
run: |
mkdir build
cd build
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_INSTALL_PREFIX=/usr \
-DTTK_BUILD_PARAVIEW_PLUGINS=TRUE \
-DTTK_BUILD_VTK_WRAPPERS=TRUE \
-DTTK_BUILD_STANDALONE_APPS=TRUE \
-DTTK_ENABLE_KAMIKAZE=TRUE \
-DTTK_ENABLE_MPI=TRUE \
-DTTK_ENABLE_CPU_OPTIMIZATION=FALSE \
-DTTK_ENABLE_SHARED_BASE_LIBRARIES=TRUE \
-DTTK_IMPLICIT_PRECONDITIONS_THRESHOLD=64*64*64 \
-GNinja \
$GITHUB_WORKSPACE
env:
CXXFLAGS: -Werror
- uses: ammaraskar/gcc-problem-matcher@master
- name: Build, install TTK & package it
run: |
cd build
cmake --build . --parallel
cpack -G DEB
sudo cmake --build . --target install
- name: Run TTK tests
uses: ./.github/actions/test-ttk-unix
- name: Upload TTK .deb package for fast install in tests
uses: actions/upload-artifact@v3
with:
name: ttk-for-tests-${{ matrix.os }}.deb
path: build/ttk.deb
- name: Archive cache
run: |
ccache -s
ccache -c
cd /home/runner
tar czf ttk-ccache.tar.gz .ccache
- name: Upload ccache archive
if: ${{ github.repository_owner == 'topology-tool-kit' && contains(github.ref, 'heads') }}
uses: actions/upload-artifact@v3
with:
name: ttk-ccache-${{ matrix.os }}
path: /home/runner/ttk-ccache.tar.gz
retention-days: 2
# ------------------#
# Tests on ttk-data #
# ------------------#
run-python-screenshots-tests:
runs-on: ${{ matrix.os }}
needs: test-build-ubuntu
strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04]
testSet: [pyTests, screenshotTests]
steps:
- name: Install Ubuntu dependencies
run: |
sudo apt update
sudo python3 -m pip install scikit-learn
- name: Fetch TTK-ParaView headless Debian package
run: |
wget -O ttk-paraview-headless.deb \
https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless-${{ matrix.os }}.deb
- name: Install ParaView .deb
run: |
sudo apt install ./ttk-paraview-headless.deb
- name: Fetch TTK packages we just built in the test-build-ubuntu job
uses: actions/download-artifact@v3
with:
name: ttk-for-tests-${{ matrix.os }}.deb
- name: Installing the previously built ttk package
run: |
sudo apt install ./ttk.deb
- uses: actions/checkout@v3
with:
repository: "topology-tool-kit/ttk-data"
ref: "dev"
path: "ttk-data"
name: Checkout ttk-data
- name: Resample large datasets
shell: python3 {0}
working-directory: ./ttk-data
run: |
from paraview import simple
for ds in ["ctBones.vti"]:
vti = simple.XMLImageDataReader(FileName=ds)
rsi = simple.ResampleToImage(Input=vti)
rsi.SamplingDimensions = [128, 128, 128]
simple.SaveData(ds, rsi)
- name: Run ttk-data states [NOT ENFORCED]
if: matrix.testSet == 'screenshotTests'
id: validate
continue-on-error: true
run: |
VERS=$(hostnamectl | grep "Operating System")
# Ubuntu 22.04 OOM killer also kills the parent process...
if [[ "$VERS" == *"22.04"* ]]; then
rm ttk-data/states/mergeTreeClustering.pvsm
rm ttk-data/states/mergeTreePGA.pvsm
rm ttk-data/states/mergeTreeTemporalReduction.pvsm
rm ttk-data/states/persistentGenerators_darkSky.pvsm
fi
cd ttk-data/tests
mkdir output_screenshots
if ! python3 -u validate.py; then
if [[ "$VERS" == *"22.04"* ]]; then
# weird opacity difference between the two Ubuntus
rm -f output_screenshots/clusteringKelvinHelmholtzInstabilities_1.png
rm -f output_screenshots/clusteringKelvinHelmholtzInstabilities_2.png
fi
if [ "$(ls -A output_screenshots)" ]; then
tar zcf screenshots.tar.gz output_screenshots
false
fi
fi
- name: Upload result screenshots
if: steps.validate.outcome == 'failure' && matrix.testSet == 'screenshotTests'
uses: actions/upload-artifact@v3
with:
name: screenshots-${{ matrix.os }}.tar.gz
path: ttk-data/tests/screenshots.tar.gz
retention-days: 10
- name: Run ttk-data Python scripts
if: matrix.testSet == 'pyTests'
run: |
cd ttk-data
python3 -u python/run.py
- name: Test ttk-data Python scripts results [NOT ENFORCED]
if: matrix.testSet == 'pyTests'
continue-on-error: true
run: |
cd ttk-data
cat python/res.json
diff python/hashes/${{ matrix.os }}.json python/res.json
- name: Delete TTK package artifact
uses: geekyeggo/delete-artifact@v2
# delete package only once
if: matrix.testSet == 'screenshotTests'
with:
name: ttk-for-tests-${{ matrix.os }}.deb
# -----------------#
# Test macOS build #
# -----------------#
test-build-macos:
runs-on: macos-12
if: ${{ github.repository_owner == 'topology-tool-kit' || !contains(github.ref, 'heads') }}
env:
CCACHE_DIR: /Users/runner/work/ttk/.ccache
CCACHE_MAXSIZE: 200M
steps:
- uses: actions/checkout@v3
name: Checkout TTK source code
- name: Install macOS dependencies
run: |
# ParaView dependencies
brew install --cask xquartz
brew install wget llvm libomp ninja python open-mpi
# TTK dependencies
brew install boost eigen graphviz numpy websocketpp ccache
# prepend ccache to system path
echo "$(brew --prefix)/opt/ccache/libexec" >> $GITHUB_PATH
- name: Install optional dependencies
uses: ./.github/actions/install-deps-unix
- name: Fetch archived ccache
continue-on-error: true
run: |
wget https://github.com/topology-tool-kit/ttk/releases/download/ccache/ttk-ccache-macOS.tar.gz
- name: Decompress ccache archive
continue-on-error: true
run: |
tar xzf ttk-ccache-macOS.tar.gz
rm -rf /Users/runner/work/ttk/.ccache
mv .ccache /Users/runner/work/ttk/
- name: Fetch TTK-ParaView headless macOS binary archive
run: |
wget https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless.tar.gz
- name: Install ParaView
run: |
tar xzf ttk-paraview-headless.tar.gz
sudo cp -r ttk-paraview/* /usr/local
pvpython -m pip install scikit-learn
- name: Set compilers as environment variables
run: |
echo "CC=$(brew --prefix llvm)/bin/clang" >> $GITHUB_ENV
echo "CXX=$(brew --prefix llvm)/bin/clang++" >> $GITHUB_ENV
- name: Create & configure TTK build directory
run: |
mkdir build
cd build
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DTTK_BUILD_PARAVIEW_PLUGINS=TRUE \
-DTTK_BUILD_VTK_WRAPPERS=TRUE \
-DTTK_BUILD_STANDALONE_APPS=TRUE \
-DTTK_ENABLE_KAMIKAZE=TRUE \
-DTTK_ENABLE_MPI=TRUE \
-DTTK_ENABLE_CPU_OPTIMIZATION=FALSE \
-DTTK_ENABLE_SHARED_BASE_LIBRARIES=TRUE \
-DTTK_IMPLICIT_PRECONDITIONS_THRESHOLD=64*64*64 \
-GNinja \
$GITHUB_WORKSPACE
env:
CXXFLAGS: -Werror
- uses: ammaraskar/gcc-problem-matcher@master
- name: Build & install TTK
run: |
cd build
cmake --build . --parallel
sudo cmake --build . --target install
- name: Set PYTHONPATH for macOS pvpython
run: |
# pvpython does not embed the correct PYTHONPATH
echo "PYTHONPATH=/usr/local/lib/python3.11/site-packages:$PYTHONPATH" >> $GITHUB_ENV
- name: Run TTK tests
uses: ./.github/actions/test-ttk-unix
- uses: actions/checkout@v3
with:
repository: "topology-tool-kit/ttk-data"
ref: "dev"
path: "ttk-data"
name: Checkout ttk-data
- name: Run ttk-data states [NOT ENFORCED]
if: false
id: validate
continue-on-error: true
run: |
cd ttk-data/tests
mkdir output_screenshots
python3 -u validate.py || (tar zcf screenshots.tar.gz output_screenshots && false)
env:
PV_PLUGIN_PATH: /usr/local/bin/plugins/TopologyToolKit
- name: Upload result screenshots
if: steps.validate.outcome == 'failure'
uses: actions/upload-artifact@v3
with:
name: screenshots-macOS.tar.gz
path: ttk-data/tests/screenshots.tar.gz
retention-days: 10
- name: Run ttk-data Python scripts
run: |
cd ttk-data
pvpython -u python/run.py
env:
PV_PLUGIN_PATH: /usr/local/bin/plugins/TopologyToolKit
- name: Test ttk-data Python scripts results [NOT ENFORCED]
continue-on-error: true
run: |
cd ttk-data
cat python/res.json
diff python/hashes/macOS.json python/res.json
- name: Archive cache
run: |
ccache -s
ccache -c
cd /Users/runner/work/ttk
tar czf ttk-ccache.tar.gz .ccache
- name: Upload ccache archive
if: ${{ github.repository_owner == 'topology-tool-kit' && contains(github.ref, 'heads') }}
uses: actions/upload-artifact@v3
with:
name: ttk-ccache-macOS
path: /Users/runner/work/ttk/ttk-ccache.tar.gz
retention-days: 2
# ------------------ #
# Test Windows build #
# ------------------ #
test-build-windows:
runs-on: windows-2022
if: ${{ github.repository_owner == 'topology-tool-kit' || !contains(github.ref, 'heads') }}
env:
PV_DIR: C:\Program Files\TTK-ParaView
TTK_DIR: C:\Program Files (x86)\ttk
CONDA_ROOT: C:\Miniconda
SCCACHE_CACHE_SIZE: 200M
steps:
- uses: actions/checkout@v3
name: Checkout TTK source code
- uses: s-weigand/setup-conda@v1
- name: Install dependencies with conda
shell: bash
run: |
conda install -c conda-forge boost glew eigen spectralib zfp \
scikit-learn openmp graphviz ninja websocketpp sccache=0.4.2 python=3.10 zlib
# add sccache to PATH
echo "$CONDA_ROOT/bin" >> $GITHUB_PATH
# add TTK & ParaView install folders to PATH
echo "$PV_DIR/bin" >> $GITHUB_PATH
echo "$TTK_DIR/bin" >> $GITHUB_PATH
- name: Remove hosted Python
shell: bash
run: |
rm -rf C:/hostedtoolcache/windows/Python
- name: Fetch archived ccache
continue-on-error: true
run: |
curl.exe -OL https://github.com/topology-tool-kit/ttk/releases/download/ccache/ttk-sccache-windows.tar.gz
- name: Decompress ccache archive
continue-on-error: true
shell: bash
run: |
tar xzf ttk-sccache-windows.tar.gz
mkdir -p /c/Users/runneradmin/AppData/Local/Mozilla/sccache
mv cache /c/Users/runneradmin/AppData/Local/Mozilla/sccache
- name: Fetch TTK-ParaView headless Windows installer
run: |
curl.exe -OL https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless.exe
- name: Install ParaView
shell: cmd
run: |
ttk-paraview-headless.exe /S
- name: Create & configure TTK build directory
shell: cmd
run: |
set CMAKE_PREFIX_PATH=%CONDA_ROOT%\Library\lib\cmake;%CONDA_ROOT%\Library\share\eigen3\cmake;%CONDA_ROOT%\Library\cmake;%ProgramFiles%\TTK-ParaView\lib\cmake
set CC=clang-cl.exe
set CXX=clang-cl.exe
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
sccache --stop-server
sccache --start-server
mkdir build
cd build
cmake ^
-DCMAKE_BUILD_TYPE=Release ^
-DCMAKE_POLICY_DEFAULT_CMP0092=NEW ^
-DBUILD_SHARED_LIBS:BOOL=TRUE ^
-DCMAKE_C_COMPILER_LAUNCHER=sccache ^
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache ^
-DPython3_ROOT_DIR="%CONDA_ROOT%" ^
-DGraphviz_INCLUDE_DIR="%CONDA_ROOT%\Library\include\graphviz" ^
-DGraphviz_CDT_LIBRARY="%CONDA_ROOT%\Library\lib\cdt.lib" ^
-DGraphviz_GVC_LIBRARY="%CONDA_ROOT%\Library\lib\gvc.lib" ^
-DGraphviz_CGRAPH_LIBRARY="%CONDA_ROOT%\Library\lib\cgraph.lib" ^
-DGraphviz_PATHPLAN_LIBRARY="%CONDA_ROOT%\Library\lib\pathplan.lib" ^
-DTTK_BUILD_PARAVIEW_PLUGINS=TRUE ^
-DTTK_BUILD_VTK_WRAPPERS=TRUE ^
-DTTK_BUILD_STANDALONE_APPS=TRUE ^
-DTTK_ENABLE_KAMIKAZE=TRUE ^
-DTTK_ENABLE_OPENMP=TRUE ^
-DTTK_ENABLE_CPU_OPTIMIZATION=FALSE ^
-DTTK_ENABLE_SHARED_BASE_LIBRARIES=TRUE ^
-DTTK_IMPLICIT_PRECONDITIONS_THRESHOLD=64*64*64 ^
-GNinja ^
..
env:
CXXFLAGS: -Werror
- name: Fix clang-cl OpenMP flags in build.ninja
shell: bash
run: |
sed -i 's/-Xclang -fopenmp/-openmp/' build/build.ninja
- name: Build & install TTK
shell: cmd
run: |
cd build
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
cmake --build . --config Release --parallel --target install
- name: Test C++ example
shell: cmd
run: |
set CC=clang-cl.exe
set CXX=clang-cl.exe
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
set CMAKE_PREFIX_PATH=%TTK_DIR%\lib\cmake;%PV_DIR%\lib\cmake
cd %GITHUB_WORKSPACE%\examples\c++
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -GNinja ..
cmake --build . --config Release --parallel
ttkExample-c++.exe -i ..\..\data\inputData.off
- name: Test VTK-C++ example
shell: cmd
run: |
set CC=clang-cl.exe
set CXX=clang-cl.exe
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
set CMAKE_PREFIX_PATH=%TTK_DIR%\lib\cmake;%PV_DIR%\lib\cmake
cd %GITHUB_WORKSPACE%\examples\vtk-c++
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -GNinja ..
cmake --build . --config Release --parallel
ttkExample-vtk-c++.exe -i ..\..\data\inputData.vtu
- name: Test Python example [TEMPORARILY NOT ENFORCED]
continue-on-error: true
shell: cmd
run: |
set PYTHONPATH=%PV_DIR%\bin\Lib\site-packages;%TTK_DIR%\bin\Lib\site-packages;%CONDA_ROOT%\Lib
cd %GITHUB_WORKSPACE%\examples\python
python example.py ..\data\inputData.vtu
- name: Test pvpython example
shell: cmd
run: |
set PYTHONPATH=%CONDA_ROOT%\Lib
set PV_PLUGIN_PATH=%TTK_DIR%\bin\plugins
cd %GITHUB_WORKSPACE%\examples\pvpython
pvpython.exe example.py ..\data\inputData.vtu
- name: Test standalone
shell: cmd
run: |
cd %GITHUB_WORKSPACE%\examples\vtk-c++
ttkHelloWorldCmd.exe -i %GITHUB_WORKSPACE%\examples\data\inputData.vtu
- uses: actions/checkout@v3
with:
repository: "topology-tool-kit/ttk-data"
ref: "dev"
path: "ttk-data"
name: Checkout ttk-data
- name: Resample large datasets
shell: cmd
run: |
set PYTHONPATH=%CONDA_ROOT%\Lib
set PV_PLUGIN_PATH=%TTK_DIR%\bin\plugins
cd ttk-data
pvpython.exe python\resample.py ctBones.vti 128 128 128 ctBones.vti
- name: Run ttk-data Python scripts
shell: cmd
run: |
set PYTHONPATH=%PV_DIR%\bin\Lib\site-packages;%TTK_DIR%\bin\Lib\site-packages;%CONDA_ROOT%\Lib;%CONDA_ROOT%\DLLs
set PV_PLUGIN_PATH=%TTK_DIR%\bin\plugins
cd ttk-data
pvpython.exe -u python\run.py
- name: Test ttk-data Python scripts results [NOT ENFORCED]
continue-on-error: true
shell: cmd
run: |
cd ttk-data
type python\res.json
FC python\hashes\windows.json python\res.json
- name: Archive cache
shell: bash
run: |
sccache -s
cd /c/Users/runneradmin/AppData/Local/Mozilla/sccache
tar czf ttk-sccache.tar.gz cache
- name: Upload sccache archive
if: ${{ github.repository_owner == 'topology-tool-kit' && contains(github.ref, 'heads') }}
uses: actions/upload-artifact@v3
with:
name: ttk-sccache-windows
path: C:\Users\runneradmin\AppData\Local\Mozilla\sccache\ttk-sccache.tar.gz
retention-days: 2
# ---------------------------------------- #
# Upload ccache archives as release assets #
# ---------------------------------------- #
ccache-release:
runs-on: ubuntu-latest
needs: [test-build-ubuntu, test-build-macos, test-build-windows]
# trigger job only on the "topology-tool-kit" repository in case
# of a branch push (pull requests and tags not affected)
if: ${{ github.repository_owner == 'topology-tool-kit' && contains(github.ref, 'heads') }}
steps:
- name: Delete previous release
uses: actions/github-script@v6
continue-on-error: true
with:
script: |
const { owner, repo } = context.repo
const { data: { id } } = await github.rest.repos.getReleaseByTag({
owner,
repo,
tag: "ccache"
})
await github.rest.repos.deleteRelease({ owner, repo, release_id: id })
- name: Create Release
uses: actions/github-script@v6
with:
script: |
const { owner, repo } = context.repo
await github.rest.repos.createRelease({
owner,
repo,
tag_name: "ccache",
name: "ccache archives",
body: "Holds ccache archives to speed up build jobs",
draft: false,
prerelease: true
})
- name: Fetch all uploaded artifacts
uses: actions/download-artifact@v3
- name: Upload Ubuntu Focal .deb as Release Asset
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ccache
file: ttk-ccache-ubuntu-20.04/ttk-ccache.tar.gz
asset_name: ttk-ccache-ubuntu-20.04.tar.gz
- name: Upload Ubuntu Jammy .deb as Release Asset
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ccache
file: ttk-ccache-ubuntu-22.04/ttk-ccache.tar.gz
asset_name: ttk-ccache-ubuntu-22.04.tar.gz
- name: Upload .pkg as Release Asset
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ccache
file: ttk-ccache-macOS/ttk-ccache.tar.gz
asset_name: ttk-ccache-macOS.tar.gz
- name: Upload sccache Windows archive
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ccache
file: ttk-sccache-windows/ttk-sccache.tar.gz
asset_name: ttk-sccache-windows.tar.gz
- name: Delete ccache artifacts
uses: geekyeggo/delete-artifact@v2
with:
name: |
ttk-*ccache*