From 74a35035c7dddbaa233ed120b9cb2ac708cbceae Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 20 Jun 2021 16:54:25 -0400 Subject: [PATCH 1/6] Add installreqs command in configure.sh --- VERSION | 2 +- configure.sh | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index da6b0a8f1..35d16fb1a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5.6 +2.5.7 diff --git a/configure.sh b/configure.sh index b57a44024..a5727d3ab 100755 --- a/configure.sh +++ b/configure.sh @@ -97,8 +97,8 @@ installruntimedepsonly() { installdepsfromsnapcraft runtime openmvs } - -install() { + +installreqs() { cd /code ## Set up library paths @@ -123,6 +123,10 @@ install() { if [ ! -z "$GPU_INSTALL" ]; then pip install --ignore-installed -r requirements.gpu.txt fi +} + +install() { + installreqs if [ ! -z "$PORTABLE_INSTALL" ]; then echo "Replacing g++ and gcc with our scripts for portability..." @@ -176,7 +180,7 @@ clean() { usage() { echo "Usage:" - echo "bash configure.sh [nproc]" + echo "bash configure.sh [nproc]" echo "Subcommands:" echo " install" echo " Installs all dependencies and modules for running OpenDroneMap" @@ -186,6 +190,8 @@ usage() { echo " Removes SuperBuild and build modules, then re-installs them. Note this does not update OpenDroneMap to the latest version. " echo " uninstall" echo " Removes SuperBuild and build modules. Does not uninstall dependencies" + echo " installreqs" + echo " Only installs the requirements (does not build SuperBuild)" echo " clean" echo " Cleans the SuperBuild directory by removing temporary files. " echo " help" @@ -193,7 +199,7 @@ usage() { echo "[nproc] is an optional argument that can set the number of processes for the make -j tag. By default it uses $(nproc)" } -if [[ $1 =~ ^(install|installruntimedepsonly|reinstall|uninstall|clean)$ ]]; then +if [[ $1 =~ ^(install|installruntimedepsonly|reinstall|uninstall|installreqs|clean)$ ]]; then RUNPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" "$1" else From 084038543eecb178e3851df6d276b14f4c091d24 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Jun 2021 10:33:42 -0400 Subject: [PATCH 2/6] Update poissonrecon, add retries --- SuperBuild/CMakeLists.txt | 2 +- opendm/mesh.py | 62 ++++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index dd7f033b2..af8a8701f 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -169,7 +169,7 @@ else() endif() externalproject_add(poissonrecon GIT_REPOSITORY https://github.com/OpenDroneMap/PoissonRecon.git - GIT_TAG 250 + GIT_TAG 257 PREFIX ${SB_BINARY_DIR}/PoissonRecon SOURCE_DIR ${SB_SOURCE_DIR}/PoissonRecon UPDATE_COMMAND "" diff --git a/opendm/mesh.py b/opendm/mesh.py index d1cc3e274..5af3d395e 100644 --- a/opendm/mesh.py +++ b/opendm/mesh.py @@ -4,6 +4,7 @@ from opendm import system from opendm import log from opendm import context +from opendm import concurrency from scipy import signal import numpy as np @@ -145,33 +146,52 @@ def screened_poisson_reconstruction(inPointCloud, outMesh, depth = 8, samples = # ext = .ply outMeshDirty = os.path.join(mesh_path, "{}.dirty{}".format(basename, ext)) + if os.path.isfile(outMeshDirty): + os.remove(outMeshDirty) # Since PoissonRecon has some kind of a race condition on ppc64el, and this helps... if platform.machine() == 'ppc64le': log.ODM_WARNING("ppc64le platform detected, forcing single-threaded operation for PoissonRecon") threads = 1 - - poissonReconArgs = { - 'bin': context.poisson_recon_path, - 'outfile': outMeshDirty, - 'infile': inPointCloud, - 'depth': depth, - 'samples': samples, - 'pointWeight': pointWeight, - 'threads': threads, - 'verbose': '--verbose' if verbose else '' - } - # Run PoissonRecon - system.run('"{bin}" --in "{infile}" ' - '--out "{outfile}" ' - '--depth {depth} ' - '--pointWeight {pointWeight} ' - '--samplesPerNode {samples} ' - '--threads {threads} ' - '--bType 2 ' - '--linearFit ' - '{verbose}'.format(**poissonReconArgs)) + while True: + poissonReconArgs = { + 'bin': context.poisson_recon_path, + 'outfile': outMeshDirty, + 'infile': inPointCloud, + 'depth': depth, + 'samples': samples, + 'pointWeight': pointWeight, + 'threads': int(threads), + 'memory': int(concurrency.get_max_memory_mb(4, 0.8) // 1024), + 'verbose': '--verbose' if verbose else '' + } + + # Run PoissonRecon + system.run('"{bin}" --in "{infile}" ' + '--out "{outfile}" ' + '--depth {depth} ' + '--pointWeight {pointWeight} ' + '--samplesPerNode {samples} ' + '--threads {threads} ' + '--maxMemory {memory} ' + '--bType 2 ' + '--linearFit ' + '{verbose}'.format(**poissonReconArgs)) + + if os.path.isfile(outMeshDirty): + break # Done! + else: + + # PoissonRecon will sometimes fail due to race conditions + # on certain machines, especially on Windows + threads //= 2 + + if threads < 1: + break + else: + log.ODM_WARNING("PoissonRecon failed with %s threads, let's retry with %s..." % (threads, threads // 2)) + # Cleanup and reduce vertex count if necessary cleanupArgs = { From edf8d883bcff2b12f2eba6fa1da89418ca4cba4a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Jun 2021 11:11:41 -0400 Subject: [PATCH 3/6] Update OpenSfM --- SuperBuild/cmake/External-OpenSfM.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SuperBuild/cmake/External-OpenSfM.cmake b/SuperBuild/cmake/External-OpenSfM.cmake index 68e47d58c..19772065b 100644 --- a/SuperBuild/cmake/External-OpenSfM.cmake +++ b/SuperBuild/cmake/External-OpenSfM.cmake @@ -19,7 +19,7 @@ ExternalProject_Add(${_proj_name} #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR} GIT_REPOSITORY https://github.com/OpenDroneMap/OpenSfM/ - GIT_TAG 256 + GIT_TAG 257 #--Update/Patch step---------- UPDATE_COMMAND git submodule update --init --recursive #--Configure step------------- From 873ab47866fa4330237511e1d3daedfeed9b782e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Jun 2021 11:21:28 -0400 Subject: [PATCH 4/6] Slow build to avoid out of memory compilation --- SuperBuild/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index af8a8701f..dd76f4ce0 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -164,7 +164,7 @@ if (WIN32) set (POISSON_BUILD_CMD ${CMAKE_MAKE_PROGRAM} ${SB_SOURCE_DIR}/PoissonRecon/PoissonRecon.vcxproj /p:configuration=${CMAKE_BUILD_TYPE} /p:PlatformToolset=${CMAKE_VS_PLATFORM_TOOLSET} /p:WindowsTargetPlatformVersion=${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}) set (POISSON_BIN_PATH "x64/${CMAKE_BUILD_TYPE}/PoissonRecon.exe") else() - set (POISSON_BUILD_CMD make -j${nproc} poissonrecon) + set (POISSON_BUILD_CMD make poissonrecon) set (POISSON_BIN_PATH "Linux/PoissonRecon") endif() externalproject_add(poissonrecon From 2e5f5ef47c9bd4fe53edbd74ff4a7cbe22a0d24a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Jun 2021 14:35:49 -0400 Subject: [PATCH 5/6] Moar swap space for builds --- .github/workflows/publish-docker-and-wsl.yaml | 4 ++++ .github/workflows/publish-docker-gpu.yaml | 4 ++++ .github/workflows/publish-snap.yml | 4 ++++ .github/workflows/test-build-prs.yaml | 4 ++++ SuperBuild/CMakeLists.txt | 2 +- 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker-and-wsl.yaml b/.github/workflows/publish-docker-and-wsl.yaml index dc28f181e..45618e6d2 100644 --- a/.github/workflows/publish-docker-and-wsl.yaml +++ b/.github/workflows/publish-docker-and-wsl.yaml @@ -13,6 +13,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 12 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/.github/workflows/publish-docker-gpu.yaml b/.github/workflows/publish-docker-gpu.yaml index f0c07dcb6..06f284b54 100644 --- a/.github/workflows/publish-docker-gpu.yaml +++ b/.github/workflows/publish-docker-gpu.yaml @@ -13,6 +13,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 12 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/.github/workflows/publish-snap.yml b/.github/workflows/publish-snap.yml index 6cc29a5f1..0b3309814 100644 --- a/.github/workflows/publish-snap.yml +++ b/.github/workflows/publish-snap.yml @@ -17,6 +17,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 12 - name: Build id: build uses: diddlesnaps/snapcraft-multiarch-action@v1 diff --git a/.github/workflows/test-build-prs.yaml b/.github/workflows/test-build-prs.yaml index fbf42ed3a..0cd2618c7 100644 --- a/.github/workflows/test-build-prs.yaml +++ b/.github/workflows/test-build-prs.yaml @@ -9,6 +9,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 12 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index dd76f4ce0..af8a8701f 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -164,7 +164,7 @@ if (WIN32) set (POISSON_BUILD_CMD ${CMAKE_MAKE_PROGRAM} ${SB_SOURCE_DIR}/PoissonRecon/PoissonRecon.vcxproj /p:configuration=${CMAKE_BUILD_TYPE} /p:PlatformToolset=${CMAKE_VS_PLATFORM_TOOLSET} /p:WindowsTargetPlatformVersion=${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}) set (POISSON_BIN_PATH "x64/${CMAKE_BUILD_TYPE}/PoissonRecon.exe") else() - set (POISSON_BUILD_CMD make poissonrecon) + set (POISSON_BUILD_CMD make -j${nproc} poissonrecon) set (POISSON_BIN_PATH "Linux/PoissonRecon") endif() externalproject_add(poissonrecon From bdd83dcee63cef3670a03206232a4c9f0d7043db Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Jun 2021 17:11:54 -0400 Subject: [PATCH 6/6] Set swap space for snap build --- .github/workflows/test-build-prs.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-build-prs.yaml b/.github/workflows/test-build-prs.yaml index 0cd2618c7..d1b7f7de2 100644 --- a/.github/workflows/test-build-prs.yaml +++ b/.github/workflows/test-build-prs.yaml @@ -33,6 +33,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 12 - name: Build id: build uses: diddlesnaps/snapcraft-multiarch-action@v1