Merge pull request #11 from florisboard/add-auto-toolchain-setup-script

Add auto-toolchain setup script & Require CMake 3.28+

.github/workflows/build.yml

@@ -25,38 +25,18 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Set up clang-16 dependencies & environment
-        run: |
-          sudo apt-get install -y libncurses5
-          echo "llvm_toolchain=$HOME/clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04" >> "$GITHUB_ENV"
-      - name: Cache clang-16
-        id: restore-clang
-        uses: actions/cache@v3
-        with:
-          path: "${{ env.llvm_toolchain }}"
-          key: clang-download-ubuntu-22.04
-      - name: Download and extract clang-16
-        if: ${{ steps.restore-clang.outputs.cache-hit != 'true' }}
-        run: |
-          cd $HOME
-          wget -q https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.3/clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04.tar.xz
-          tar -xf clang+llvm-16.0.3-x86_64-linux-gnu-ubuntu-22.04.tar.xz
-
       - name: "Set up CMake and Ninja"
         uses: lukka/get-cmake@latest
-        with:
-          cmakeVersion: "~3.26.0"
 
+      - name: "Toolchain setup"
+        run: |
+          $(pwd)/setup-toolchain.sh
       - name: "CMake: Configure release"
         run: |
-          cmake --preset=release $(pwd) \
-            -DCMAKE_C_COMPILER="${{ env.llvm_toolchain }}/bin/clang" \
-            -DCMAKE_CXX_COMPILER="${{ env.llvm_toolchain }}/bin/clang++"
+          $(pwd)/cmake.sh --preset=release $(pwd)
       - name: "CMake: Build release"
         run: |
-          cmake --build --preset=release $(pwd) \
-            -DCMAKE_C_COMPILER="${{ env.llvm_toolchain }}/bin/clang" \
-            -DCMAKE_CXX_COMPILER="${{ env.llvm_toolchain }}/bin/clang++"
+          $(pwd)/cmake.sh --build --preset=release $(pwd)
 
       - name: "Upload artifacts"
         uses: actions/upload-artifact@v3

.gitignore

@@ -33,6 +33,7 @@
 
 # CMake files
 build/
+buildtools/
 CMakeLists.txt.user
 CMakeCache.txt
 CMakeFiles
@@ -50,3 +51,6 @@ CMakeUserPresets.json
 # Python
 *.pyc
 __pycache__
+
+# dotenv
+.env.buildtools

CMakeLists.txt

@@ -1,9 +1,18 @@
 if(NOT ANDROID)
-    cmake_minimum_required(VERSION 3.26)
+    cmake_minimum_required(VERSION 3.28)
 else()
     cmake_minimum_required(VERSION 3.22)
 endif()
 
+# Set up LLVM toolchain
+if(LLVM_TOOLCHAIN)
+    set(CMAKE_C_COMPILER "${LLVM_TOOLCHAIN}/bin/clang")
+    set(CMAKE_CXX_COMPILER "${LLVM_TOOLCHAIN}/bin/clang++")
+else()
+    get_filename_component(LLVM_TOOLCHAIN ${CMAKE_CXX_COMPILER} DIRECTORY)
+    get_filename_component(LLVM_TOOLCHAIN ${LLVM_TOOLCHAIN} DIRECTORY)
+endif()
+
 project(FlorisNLP VERSION 0.1.0)
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
@@ -26,10 +35,6 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-# Get llvm toolchain path
-get_filename_component(LLVM_TOOLCHAIN ${CMAKE_CXX_COMPILER} DIRECTORY)
-get_filename_component(LLVM_TOOLCHAIN ${LLVM_TOOLCHAIN} DIRECTORY)
-
 # We MUST disable json ranges support else this project will not compile properly with clang-16.
 # See also: external/nlohmann-json/docs/mkdocs/docs/api/macros/json_has_ranges.md
 if(NOT ANDROID)

CMakeModules/CxxModules.cmake

@@ -1,7 +1,3 @@
-include(ClangModules)
-
-set(CMake_TEST_CXXModules_UUID ${CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API})
-
 function(target_module_sources target_name target_type mod_src1)
     target_sources(${target_name} ${target_type} FILE_SET cxx_modules TYPE CXX_MODULES FILES ${mod_src1} ${ARGN})
 endfunction()

CMakePresets.json

@@ -2,7 +2,7 @@
   "version": 3,
   "cmakeMinimumRequired": {
     "major": 3,
-    "minor": 26,
+    "minor": 28,
     "patch": 0
   },
   "configurePresets": [
@@ -16,7 +16,6 @@
         "ICU_BUILD_FROM_SOURCE": false,
         "CMAKE_C_COMPILER": "clang",
         "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API": "2182bf5c-ef0d-489a-91da-49dbc3090d2a",
         "CMAKE_ARCHIVE_OUTPUT_DIRECTORY": "${sourceDir}/build/${presetName}/bin",
         "CMAKE_LIBRARY_OUTPUT_DIRECTORY": "${sourceDir}/build/${presetName}/bin",
         "CMAKE_RUNTIME_OUTPUT_DIRECTORY": "${sourceDir}/build/${presetName}/bin"

README.md

@@ -26,13 +26,14 @@ To be able to compile this project on your PC, you must run a supported host sys
 System | Submodule build | Standalone build | Notes
 ---|---|---|---
 Windows 7/8/10/11 | ❌ | ❌ | -
-Windows 10/11 with WSL2 | ✅ | ⚠️ | tested with distro:<br>Ubuntu 22.10
+Windows 10/11 with WSL2 | ✅ | ⚠️ | tested with distro:<br>Ubuntu 23.10
 MacOS | ❔ | ❔ | untested but should be supported; please provide feedback
 Debian 11.0 | ⚠️ | ❌ | submodule: `python3` package is outdated in the package repository
 Debian 12.0+ | ✅ | ⚠️ | standalone: `cmake` and `clang` packages are outdated in the package repository
 Ubuntu 22.04 | ✅ | ❌ | -
 Ubuntu 22.10 | ✅ | ⚠️ | standalone: `cmake` and `clang` packages are outdated in the package repository
-Ubuntu 23.04+ | ✅ | ⚠️ | standalone: `cmake` and `clang` packages are outdated in the package repository
+Ubuntu 23.04 | ✅ | ⚠️ | standalone: `cmake` and `clang` packages are outdated in the package repository
+Ubuntu 23.10+ | ✅ | ⚠️ | standalone: `cmake` and `clang` packages are outdated in the package repository
 Other Linux systems | ❔ | ❔ | try yourself
 
 ### Submodule build (targeting `Android`)
@@ -41,14 +42,14 @@ Requirements if you compile NLP as a submodule for the main FlorisBoard project:
 
 - Android SDK
 - Java 17
-- Android NDK r25
+- Android NDK r25 or newer
 - CMake 3.22+
 - Ninja 1.10+
-- GNU make 3.80+
-  - MUST be GNU make and not some other variation of make or the ICU build will fail!!
 - Clang 14.x+ & libc++ (bundled with Android NDK)
 - Python 3.10+
 - Git
+- Optional: GNU make 3.80+
+  - Only required if `ICU_BUILD_FROM_SOURCE` is enabled
 
 If you have trouble installing the requirements or don't know how to install some you can
 also refer to [this excellent guide](https://github.com/florisboard/florisboard/issues/2218#issuecomment-1572763444) written by [@Thithic](https://github.com/Thithic), which guides you step by step in setting up the requirements on Ubuntu 22.04.
@@ -63,13 +64,16 @@ FlorisBoard.
 
 Requirements if you compile NLP and NLP tools as a standalone utility for desktop:
 
-- CMake 3.26+
+- CMake 3.28+
 - Ninja 1.11+
-- GNU make 3.80+
-  - MUST be GNU make and not some other variation of make or the ICU build will fail!!
 - Clang 16.x+ (see below if your distro does not have version 16 yet)
-- Package `libc++-dev` and `libc++abi-dev` (version 16.x+)
+  - The following packages are needed (all version 16.x+):
+    - `clang clang-tools libc++-dev libc++abi-dev`
+  - *Tip: the default clang-packages my not be clang 16.x+ yet, in this case you can try to install*
+    - `clang-16 clang-tools-16 libc++-16-dev libc++abi-16-dev`
 - Git
+- Optional: GNU make 3.80+
+  - Only required if `ICU_BUILD_FROM_SOURCE` is enabled
 
 #### Initializing the local source repository
 
@@ -80,7 +84,17 @@ cd nlp
 git submodule update --init --recursive
 ```
 
-#### Set up clang compiler
+#### Set up cmake/clang compiler (Ubuntu 22.04+ only)
+
+Thetoolchain setup is automated for Ubuntu 22.04+ and can be invoked like this:
+
+```shell
+./setup-toolchain.sh
+```
+
+After a successful run of the script, you can use `./cmake.sh` in-place of the normal `cmake` command.
+
+#### Set up cmake/clang compiler (Manual)
 
 Before you can build this project for Desktop targets you need to set up the clang compiler. First check which version
 you have installed:
@@ -98,18 +112,19 @@ Great! You do not have to set up anything else, and you can skip to the project
 <details>
 <summary>The reported version is 15.x or older</summary>
 
-In this case you do not have a supported version of clang installed, and we need to download and integrate the compiler
-manually. Head to https://github.com/llvm/llvm-project/releases and download the appropriate prebuilt llvm-project for
-your system. Below example assumes you are on Ubuntu 22.04.
+In this case you do not have a supported version of clang installed, and we need to download and integrate the compiler manually. Head to https://github.com/llvm/llvm-project/releases and download the appropriate prebuilt llvm-project for your system. Below example assumes you are on Ubuntu 22.04 or newer.
 
 ```shell
-# Download clang 16.0.x for Ubuntu 22.04
+# Download clang 17.0.x for Ubuntu 22.04+
+wget https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.5/clang+llvm-17.0.5-x86_64-linux-gnu-ubuntu-22.04.tar.xz
+tar -xf clang+llvm-17.0.5-x86_64-linux-gnu-ubuntu-22.04.tar.xz
+# Alternatively you can also download clang 16.0.x for Ubuntu 22.04+
+#   not recommended anymore due to minor issues between cmake&clang, however still supported
 wget https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz
 tar -xf clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz
 ```
 
-After this we need to change the compiler path to the downloaded one, else the build will fail. To change it, open
-[CMakePresets.json](CMakePresets.json) in a text editor and change the C/CXX compiler vars like so:
+After this we need to change the compiler path to the downloaded one, else the build will fail. To change it, open [CMakePresets.json](CMakePresets.json) in a text editor and change the C/CXX compiler vars like so:
 
 ```
     ...
@@ -120,14 +135,6 @@ After this we need to change the compiler path to the downloaded one, else the b
 
 </details>
 
-#### Adjusting the `CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API` UUID
-
-Atm we need to adjust the `CMAKE_EXPERIMENTAL_CXX_MODULE_CMAKE_API` UUID right below the compiler path fields in
-[CMakePresets.json](CMakePresets.json), depending on the CMake version you use (UUID committed in this branch is for
-CMake 3.26.x). See https://github.com/Kitware/CMake/blob/v3.26.0-rc6/Help/dev/experimental.rst (adjust the version tag
-in the URL) for the correct UUID. This step is also temporary and once C++ module support is stable in CMake this will
-not be needed to be adjusted anymore.
-
 #### Building
 
 ```shell

cmake.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+
+# Copyright 2023 Patrick Goldinger
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dotenv_file="$(realpath .env.buildtools)"
+
+cd "$(realpath "$(dirname "$0")")" || exit 1
+
+if [ -f "$dotenv_file" ]; then
+    # shellcheck source=/dev/null
+    . "$dotenv_file"
+    cmake_bin="${CMAKE_BIN:="cmake"}"
+    llvm_toolchain="${LLVM_TOOLCHAIN:=""}"
+else
+    cmake_bin="cmake"
+    llvm_toolchain=""
+fi
+
+is_build_command=0
+for arg in "$@"; do
+    if [ "$arg" = "--build" ]; then
+        is_build_command=1
+        break
+    fi
+done
+
+if [ "$#" -eq 0 ]; then
+    "$cmake_bin" --version
+elif [ $is_build_command -eq 1 ]; then
+    "$cmake_bin" "$@"
+else
+    "$cmake_bin" -DLLVM_TOOLCHAIN="$llvm_toolchain" "$@"
+fi
+exit $?