Merge pull request #442 from KhiopsML/dev-v10

Prepare Release 10.2.3

.cmake-format.py

@@ -1,3 +1,7 @@
+# Copyright (c) 2024 Orange. All rights reserved.
+# This software is distributed under the BSD 3-Clause-clear License, the text of which is available
+# at https://spdx.org/licenses/BSD-3-Clause-Clear.html or see the "LICENSE" file for more details.
+
 with section("format"):
     line_width = 120
     tab_size = 2

.github/actions/build-khiops/action.yml

@@ -13,7 +13,7 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Setup MPI (windows)
+    - name: Setup MPI (Windows)
       if: runner.os == 'Windows'
       uses: mpi4py/setup-mpi@v1
     - name: Setup MPI (macOS)
@@ -29,9 +29,16 @@ runs:
         sudo apt-get install openmpi-bin libopenmpi-dev
         sudo update-alternatives --set mpi /usr/bin/mpicc.openmpi
         sudo update-alternatives --set mpirun /usr/bin/mpirun.openmpi
-    - name: Setup Ninja
-      uses: ashutoshvarma/setup-ninja@v1.1
-    - name: Load Visual C++ Environment Variables (Windows only)
+    - name: Setup Ninja (Linux)
+      if: runner.os == 'Linux'
+      shell: bash
+      run: sudo apt-get install ninja-build
+    - name: Setup Ninja (macOS)
+      if: runner.os == 'macOS'
+      shell: bash
+      run: brew install ninja
+    # This step is required to have ninja and cmake in the path for windows
+    - name: Load Visual C++ Environment Variables (Windows)
       if: runner.os == 'Windows'
       shell: cmd
       run: |

.github/actions/check-tag-version/action.yml

@@ -0,0 +1,17 @@
+---
+name: Check tag/version consistency
+description: Check that the Khiops source version matches a tag
+runs:
+  using: composite
+  steps:
+    - name: Check that the Khiops source version matches the input tag
+      shell: bash
+      run: |-
+        SOURCE_VERSION=$(./scripts/khiops-version)
+        if [[ "$SOURCE_VERSION" != "${{ github.ref_name }}" ]]
+        then
+          echo "::error::Incompatible source version '$SOURCE_VERSION' with tag '${{ github.ref_name }}'. Either change the tag to match the source version or modify the KHIOPS_VERSION macro in src/Learning/KWUtils/KWKhiopsVersion.h."
+          exit 1
+        else
+          echo "Tag '${{ github.ref_name }}' OK"
+        fi

.github/actions/test-khiops-install/action.yml

@@ -1,27 +1,100 @@
 ---
 name: Test Khiops Installation
-description: Check the outputs of Khiops in the console
+description: Test khiops' scripts (khiops_env, khiops and khiops_coclustering)
 runs:
   using: composite
-  # Test if the output of Khiops in the console is as expected (no mpi or java errors)
-  # We expect only one line in stdout+stderr when running 'khiops -v'
   steps:
-    - name: Print status for debugging
-      shell: bash
-      # We force the process number to 4 because the runner has only 2 proc.
-      # With 2 proc khiops-env switches to serial and mpi is not used.
+    - name: Test that the executables are installed (Linux/macOS)
+      # We call khiops_env a first time to test the calling of _khiopsgetprocnumber
+      # then we play with KHIOPS_PROC_NUMBER:
+      # - with KHIOPS_PROC_NUMBER=2, khiops should not use MPI, we check that KHIOPS_MPI_COMMAND is empty
+      # - with KHIOPS_PROC_NUMBER=8, khiops should use MPI, we check that KHIOPS_MPI_COMMAND is not empty and KHIOPS_MPI_ERROR is empty
+      #
       # The env var OMPI_MCA_rmaps_base_oversubscribe is for openmpi it corresponds to
       # the flag --oversubscribe (Nodes are allowed to be oversubscribed)
+      # KHIOPS_MPI_VERBOSE is set to true to detect errors from MPI
+      if: runner.os != 'Windows'
+      shell: bash
       env:
-        KHIOPS_PROC_NUMBER: 4
         OMPI_MCA_rmaps_base_oversubscribe: true
+        KHIOPS_MPI_VERBOSE: true
       run: |
-        khiops-env --env
-    - name: Check Khiops output
+        echo
+        echo --- Khiops environment with KHIOPS_PROC_NUMBER not set
+        khiops_env --env
+        echo
+        export KHIOPS_PROC_NUMBER=2
+        echo --- Khiops environment with KHIOPS_PROC_NUMBER=$KHIOPS_PROC_NUMBER
+        source khiops_env --env
+        if [ ! -z "$KHIOPS_MPI_COMMAND" ]; 
+        then 
+          echo "::error::MPI is used even though there are only 2 procs available"
+          false
+        fi
+        echo
+        export KHIOPS_PROC_NUMBER=8
+        echo --- Khiops environment with KHIOPS_PROC_NUMBER=$KHIOPS_PROC_NUMBER
+        source khiops_env --env
+        if [ ! -z "$KHIOPS_MPI_ERROR" ]; 
+        then 
+          echo "::error::Unexpected MPI error: $KHIOPS_MPI_ERROR"
+          false
+        fi
+        if [ -z "$KHIOPS_MPI_COMMAND" ]; 
+        then 
+          echo "::error::KHIOPS_MPI_COMMAND is not set"
+          false
+        fi
+        echo
+        echo --- Khiops status
+        khiops -s
+        echo
+        echo --- Khiops Coclustering status
+        khiops_coclustering -s
+    - name: Test that the executables are installed (windows)
+      if: runner.os == 'Windows'
+      shell: cmd /C call {0}
+      run: |
+        echo.
+        echo --- Khiops environment with KHIOPS_PROC_NUMBER not set
+        call khiops_env --env
+        echo.
+        set KHIOPS_PROC_NUMBER=2
+        echo --- Khiops environment with KHIOPS_PROC_NUMBER=%KHIOPS_PROC_NUMBER%
+        call khiops_env --env
+        call khiops_env
+        if not "%KHIOPS_MPI_COMMAND%". == "". (
+          echo "::error::MPI is used even though there are only 2 procs available"
+          exit 1
+        )
+        set KHIOPS_PROC_NUMBER=8
+        echo.
+        echo --- Khiops environment with KHIOPS_PROC_NUMBER=%KHIOPS_PROC_NUMBER%
+        call khiops_env --env
+        call khiops_env
+        if not "%KHIOPS_MPI_ERROR%".=="". (
+          echo "::error::Unexpected MPI error: %KHIOPS_MPI_ERROR%"
+          exit 1
+        )
+        if "%KHIOPS_MPI_COMMAND%". == "". (
+          echo "::error::KHIOPS_MPI_COMMAND is not set"
+          exit 1
+        )
+        echo.
+        echo --- Khiops status
+        khiops -s
+        echo.
+        echo --- Khiops Coclustering status
+        khiops_coclustering -s
+    - name: Check Khiops output [UNIXes]
+      # Test if the output of Khiops in the console is as expected (no mpi or java errors)
+      # We expect only one line in stdout+stderr when running 'khiops -v'
+      if: runner.os != 'Windows'
       shell: bash
       env:
         KHIOPS_PROC_NUMBER: 4
         OMPI_MCA_rmaps_base_oversubscribe: true
+        KHIOPS_MPI_VERBOSE: true
       run: |
         khiops -s
         khiops -v &> output
@@ -31,7 +104,20 @@ runs:
           echo "::error::Unexpected output in khiops scripts"
           false
         fi
-    - name: Check Khiops coclustering output
+    - name: Check Khiops output [Windows]
+      if: runner.os == 'Windows'
+      shell: cmd /C call {0}
+      env:
+        KHIOPS_PROC_NUMBER: 4
+      run: |
+        khiops -s
+        for /f %%i in (' khiops -v ^| find /c /v "" ') do set "LINE_NUMBER=%%i"
+        if not "%LINE_NUMBER%" == "1" (
+          echo "::error::Unexpected output in khiops scripts"
+          exit 1
+        )
+    - name: Check Khiops coclustering output [UNIXes]
+      if: runner.os != 'Windows'
       shell: bash
       run: |-
         khiops_coclustering -s
@@ -42,14 +128,26 @@ runs:
           echo "::error::Unexpected output in khiops_coclustering scripts"
           false
         fi
-    - name: Check process number
+    - name: Check Khiops coclustering output [Windows]
+      if: runner.os == 'Windows'
+      shell: cmd /C call {0}
+      run: |
+        khiops_coclustering -s
+        for /f %%i in (' khiops_coclustering -v ^| find /c /v "" ') do set "LINE_NUMBER=%%i"
+        if not "%LINE_NUMBER%" == "1" (
+          echo "::error::Unexpected output in khiops_coclustering scripts"
+          exit 1
+        )
+    - name: Check process number [UNIXes]
+      # Test if Khiops has the right number of logical processes
+      # (it's obvious but you never know)
+      if: runner.os != 'Windows'
       shell: bash
       env:
         KHIOPS_PROC_NUMBER: 4
         OMPI_MCA_rmaps_base_oversubscribe: true
       run: |-
-        PROC_NUMBER_LIST=($(khiops -s | grep 'Logical processes on system'))
-        PROC_NUMBER=${PROC_NUMBER_LIST[-1]}
+        PROC_NUMBER=$(khiops -s | grep "Logical processes" | awk '{print $NF}')
         if [ "$PROC_NUMBER" != "$KHIOPS_PROC_NUMBER" ] ;
         then
           echo "::error::Wrong proc number ($PROC_NUMBER vs $KHIOPS_PROC_NUMBER)"

.github/actions/test-khiops-on-iris/action.yml

@@ -1,21 +1,29 @@
 ---
 name: Test Khiops on Iris
 description: Tests a Khiops installation on the Iris dataset
+inputs:
+  os-decription:
+    description: OS description (os +arch) used to name artifacts (error reports)
+    required: true
 runs:
   using: composite
   steps:
-    # Python install: we don't use the setup-python action because of the following error:
-    # python3: /lib64/libc.so.6: version `GLIBC_2.34' not found (required by python3)
     - name: Python setup for Linux
       if: runner.os == 'Linux'
       shell: bash
       run: |
-        # Detect Debian based OS
-        if [ -d "/etc/apt" ]
+        # Install Python 3 if not present (already present in conda environment)
+        if ! command -v python3 &> /dev/null
         then
-          apt-get install -y python3 > /dev/null
-        else
-          yum install -y python3.11
+          # Python install: we don't use the setup-python action because of the following error:
+          # python3: /lib64/libc.so.6: version `GLIBC_2.34' not found (required by python3)
+          # Detect Debian based OS
+          if [ -d "/etc/apt" ]
+          then
+            apt-get install -y python3 > /dev/null
+          else
+            yum install -y python3.11
+          fi
         fi
         echo "PYTHON=python3" >> "$GITHUB_ENV"
     - name: Python setup for Windows or macOS
@@ -39,19 +47,12 @@ runs:
           echo "KHIOPS_SCRIPT=khiops" >> "$GITHUB_ENV"
           echo "KHIOPS_CC_SCRIPT=khiops_coclustering" >> "$GITHUB_ENV"
         fi
-    - name: Check Khiops installation
-      shell: bash
-      env:
-        KHIOPS_PROC_NUMBER: 4
-        OMPI_MCA_rmaps_base_oversubscribe: true
-      run: |
-        "$KHIOPS_SCRIPT" -s
-        "$KHIOPS_CC_SCRIPT" -s
     - name: Run Khiops tests
       shell: bash
       env:
         KHIOPS_PROC_NUMBER: 4
         OMPI_MCA_rmaps_base_oversubscribe: true
+        KHIOPS_MPI_VERBOSE: true
       run: |
         cd test/LearningTest/TestKhiops/Standard/Iris/
         "$KHIOPS_SCRIPT" -b -i test.prm -e results/err.txt
@@ -73,7 +74,17 @@ runs:
         $PYTHON kht_test.py ../../LearningTest/TestCoclustering/Standard/Iris check
         $PYTHON kht_apply.py ../../LearningTest/TestKhiops/Standard/Iris errors | tee /tmp/khiops-log.txt
         $PYTHON kht_apply.py ../../LearningTest/TestCoclustering/Standard/Iris errors | tee /tmp/coclustering-log.txt
+        $PYTHON kht_collect_results.py ../../LearningTest/TestKhiops/Standard/Iris $GITHUB_WORKSPACE/results/khiops --collect-type warnings
+        $PYTHON kht_collect_results.py ../../LearningTest/TestCoclustering/Standard/Iris $GITHUB_WORKSPACE/results/coclustering --collect-type warnings 
         if (grep -q error /tmp/khiops-log.txt || grep -q error /tmp/coclustering-log.txt); then
           echo "::error::Errors in Khiops run"
           false
         fi
+    - name: Upload test results
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-results-${{ inputs.os-decription}}
+        retention-days: 7
+        path: |-
+          results/

.github/workflows/build-linux-pack-containers.yml

@@ -17,6 +17,7 @@ jobs:
           - ubuntu18.04
           - ubuntu20.04
           - ubuntu22.04
+          - ubuntu24.04
     permissions:
       packages: write  # Allows writing in the container registry
     steps: