Support for lightgbm >= 4.0 (onnx#634)

* Fix issues raised with lightgbm 4.0 Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * update CI Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * remove use of eval Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * fix requirements Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * update requirements Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * fix handle Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * fix handle Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * fix scipy version Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * svm Signed-off-by: Xavier Dupre <xadupre@microsoft.com> * remove allow_failure Signed-off-by: Xavier Dupre <xadupre@microsoft.com> --------- Signed-off-by: Xavier Dupre <xadupre@microsoft.com>
xadupre · Jul 28, 2023 · b4696fb · b4696fb
1 parent e4a4745
commit b4696fb
Show file tree

Hide file tree

Showing 16 changed files with 155 additions and 236 deletions.
diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml
@@ -15,90 +15,46 @@ jobs:
   strategy:
     matrix:
 
-      Python310-1140-RT1150-xgb175:
+      Python311-1140-RT1151-xgb175:
+        python.version: '3.11'
+        ONNX_PATH: 'onnx==1.14.0' #'-i https://test.pypi.org/simple/ onnx==1.14.0rc3'
+        ONNXRT_PATH: 'onnxruntime==1.15.1'
+        COREML_PATH: NONE
+        lightgbm.version: '>=4.0'
+        xgboost.version: '>=1.7.5'
+        numpy.version: ''
+        scipy.version: ''
+
+      Python310-1140-RT1151-xgb175:
         python.version: '3.10'
         ONNX_PATH: 'onnx==1.14.0' #'-i https://test.pypi.org/simple/ onnx==1.14.0rc3'
-        ONNXRT_PATH: '-i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ ort_nightly==1.15.0.dev20230502003'
+        ONNXRT_PATH: 'onnxruntime==1.15.1'
         COREML_PATH: NONE
+        lightgbm.version: '<4.0'
         xgboost.version: '>=1.7.5'
         numpy.version: ''
+        scipy.version: ''
 
       Python310-1140-RT1140-xgb175:
         python.version: '3.10'
         ONNX_PATH: 'onnx==1.14.0' #'-i https://test.pypi.org/simple/ onnx==1.14.0rc3'
         ONNXRT_PATH: onnxruntime==1.14.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
         COREML_PATH: NONE
+        lightgbm.version: '<4.0'
         xgboost.version: '>=1.7.5'
         numpy.version: ''
-      # Python310-1130-RT1140-xgb173:
-      #   python.version: '3.10'
-      #   ONNX_PATH: 'onnx==1.13.0' #'-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.14.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: NONE
-      #   xgboost.version: '>=1.7.3'
-      #   numpy.version: ''
-      # Python310-1130-RT1131-xgb173:
-      #   python.version: '3.10'
-      #   ONNX_PATH: 'onnx==1.13.0' #'-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.13.1 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: NONE
-      #   xgboost.version: '>=1.7.3'
-      #   numpy.version: ''
-      # Python310-1120-RT1121-xgb161:
-      #   python.version: '3.10'
-      #   ONNX_PATH: 'onnx==1.12.0' #'-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.12.1 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: NONE
-      #   xgboost.version: '==1.6.1'
-      #   numpy.version: ''
-      # Python39-1120-RT1110-xgb161:
-      #   python.version: '3.9'
-      #   ONNX_PATH: 'onnx==1.12.0' #'-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.11.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: coremltools==6.3 # git+https://github.com/apple/coremltools@3.1
-      #   xgboost.version: '>=1.6.1'
-      #   numpy.version: ''
-      # Python39-1120-RT1110-xgb160:
-      #   python.version: '3.9'
-      #   ONNX_PATH: 'onnx==1.12.0' #'-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.11.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: git+https://github.com/apple/coremltools@3.1
-      #   xgboost.version: '==1.6.0'
-      #   numpy.version: ''
-      # Python39-1110-RT1110:
-      #   python.version: '3.9'
-      #   ONNX_PATH: onnx==1.11.0  # '-i https://test.pypi.org/simple/ onnx==1.9.101'
-      #   ONNXRT_PATH: onnxruntime==1.11.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: git+https://github.com/apple/coremltools@3.1
-      #   numpy.version: ''
-      # Python39-1110-RT1100-xgb120:
-      #   python.version: '3.9'
-      #   ONNX_PATH: onnx==1.11.0  # '-i https://test.pypi.org/simple/ onnx==1.9.101'
-      #   ONNXRT_PATH: onnxruntime==1.10.0
-      #   COREML_PATH: git+https://github.com/apple/coremltools@3.1
-      #   xgboost.version: '>=1.2'
-      #   numpy.version: ''
-      # Python39-1101-RT190-xgb120:
-      #   python.version: '3.9'
-      #   ONNX_PATH: onnx==1.10.1
-      #   ONNXRT_PATH: onnxruntime==1.9.0
-      #   COREML_PATH: coremltools==4.0 # git+https://github.com/apple/coremltools@3.1
-      #   xgboost.version: '>=1.2'
-      #   numpy.version: '<=1.23.5'
-      # Python39-190-RT180-xgb120:
-      #   python.version: '3.9'
-      #   ONNX_PATH: onnx==1.9.0
-      #   ONNXRT_PATH: onnxruntime==1.8.0
-      #   COREML_PATH: coremltools==4.0 # git+https://github.com/apple/coremltools@3.1
-      #   xgboost.version: '>=1.2'
-      #   numpy.version: '<=1.23.5'
-      # Python38-181-RT170-xgb120:
-      #   python.version: '3.8'
-      #   ONNX_PATH: onnx==1.8.1
-      #   ONNXRT_PATH: onnxruntime==1.7.0
-      #   COREML_PATH: coremltools==4.0 # git+https://github.com/apple/coremltools@3.1
-      #   xgboost.version: '>=1.2'
-      #   numpy.version: '<=1.23.5'
+        scipy.version: ''
+
+      Python39-1140-RT1151-xgb175-scipy180:
+        python.version: '3.9'
+        ONNX_PATH: 'onnx==1.14.0' #'-i https://test.pypi.org/simple/ onnx==1.14.0rc3'
+        ONNXRT_PATH: 'onnxruntime==1.15.1'
+        COREML_PATH: NONE
+        lightgbm.version: '>=4.0'
+        xgboost.version: '>=1.7.5'
+        numpy.version: ''
+        scipy.version: '==1.8.0'
+
 
     maxParallel: 3
 
@@ -120,16 +76,18 @@ jobs:
 
   - script: |
       python -m pip install --upgrade pip
-      pip install xgboost$(xgboost.version)
+      pip install "xgboost$(xgboost.version)"
+      pip install "lightgbm$(lightgbm.version)"
       pip install $(ONNX_PATH)
       pip install $(ONNXRT_PATH)
       pip install "numpy$(numpy.version)"
+      pip install "scipy$(scipy.version)"
     displayName: 'Install xgboost, onnxruntime'
 
   - script: |
       python -m pip install coloredlogs flatbuffers packaging sympy numpy protobuf
       python -m pip install $(ONNXRT_PATH)
-    displayName: 'Install ort-nightly'
+    displayName: 'Install onnxruntime'
 
   - script: |
       pip install flake8
@@ -170,11 +128,6 @@ jobs:
       pytest tests/sparkml --durations=0
     displayName: 'pytest - sparkml'
 
-  - script: |
-      export PYTHONPATH=.
-      pytest tests/svmlib --durations=0
-    displayName: 'pytest - svmlib'
-
   - script: |
       export PYTHONPATH=.
       pytest tests/utils --durations=0
@@ -191,6 +144,11 @@ jobs:
       pytest tests/h2o --durations=0
     displayName: 'pytest - h2o'
 
+  - script: |
+      export PYTHONPATH=.
+      pytest tests/svmlib --durations=0
+    displayName: 'pytest - svmlib'
+
   - script: |
       pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
       pip install hummingbird-ml --no-deps

diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml
@@ -15,10 +15,17 @@ jobs:
   strategy:
     matrix:
 
-      Python310-1150-RT1140:
+      Python311-1140-RT1151:
+        python.version: '3.11'
+        ONNX_PATH: 'onnx==1.14.0' # '-i https://test.pypi.org/simple/ onnx==1.14.0rc3'
+        ONNXRT_PATH: 'onnxruntime==1.15.1'
+        COREML_PATH: NONE
+        numpy.version: ''
+
+      Python310-1140-RT1151:
         python.version: '3.10'
         ONNX_PATH: 'onnx==1.14.0' # '-i https://test.pypi.org/simple/ onnx==1.14.0rc3'
-        ONNXRT_PATH: '-i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ ort_nightly==1.15.0.dev20230502003'
+        ONNXRT_PATH: 'onnxruntime==1.15.1'
         COREML_PATH: NONE
         numpy.version: ''
 
@@ -29,69 +36,6 @@ jobs:
         COREML_PATH: NONE
         numpy.version: ''
 
-      # Python310-1130-RT1140:
-      #   python.version: '3.10'
-      #   ONNX_PATH: 'onnx==1.13.0' # '-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.14.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: NONE
-      #   numpy.version: ''
-
-      # Python310-1130-RT1131:
-      #   python.version: '3.10'
-      #   ONNX_PATH: 'onnx==1.13.0' # '-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.13.1 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: NONE
-      #   numpy.version: ''
-
-      # Python39-1120-RT1110:
-      #   python.version: '3.9'
-      #   ONNX_PATH: 'onnx==1.12.0' # '-i https://test.pypi.org/simple/ onnx==1.12.0rc4'
-      #   ONNXRT_PATH: onnxruntime==1.11.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: coremltools==6.3 # git+https://github.com/apple/coremltools@3.1
-      #   numpy.version: ''
-
-      # Python39-1110-RT1110:
-      #   python.version: '3.9'
-      #   ONNX_PATH: onnx==1.11.0  # '-i https://test.pypi.org/simple/ onnx==1.9.101'
-      #   ONNXRT_PATH: onnxruntime==1.11.0 #'-i https://test.pypi.org/simple/ ort-nightly==1.11.0.dev20220311003'
-      #   COREML_PATH: coremltools==6.2  # 6.0 doesn't work
-      #   numpy.version: ''
-
-      # Python39-1110-RT190:
-      #   python.version: '3.9'
-      #   ONNX_PATH: 'onnx==1.11.0'  # '-i https://test.pypi.org/simple/ onnx==1.9.101'
-      #   ONNXRT_PATH: onnxruntime==1.10.0
-      #   COREML_PATH: coremltools==5.2 # git+https://github.com/apple/coremltools@3.1
-      #   numpy.version: ''
-
-      # Python39-1102-RT190:
-      #   python.version: '3.9'
-      #   ONNX_PATH: 'onnx==1.10.2'  # '-i https://test.pypi.org/simple/ onnx==1.9.101'
-      #   ONNXRT_PATH: onnxruntime==1.9.0
-      #   COREML_PATH: coremltools==5.2 # 5.0 doesn't work
-      #   numpy.version: '<=1.23.5'
-
-      # Python39-190-RT181:
-      #   python.version: '3.9'
-      #   ONNX_PATH: 'onnx==1.9.0'
-      #   ONNXRT_PATH: onnxruntime==1.8.1
-      #   COREML_PATH: coremltools==4.0 # git+https://github.com/apple/coremltools@3.1
-      #   numpy.version: '<=1.23.5'
-
-      # Python39-190-RT180:
-      #   python.version: '3.9'
-      #   ONNX_PATH: onnx==1.9.0
-      #   ONNXRT_PATH: onnxruntime==1.8.0
-      #   COREML_PATH: coremltools==4.0 # git+https://github.com/apple/coremltools@3.1
-      #   numpy.version: '<=1.23.5'
-
-      # Python38-181-RT170:
-      #   python.version: '3.8'
-      #   ONNX_PATH: onnx==1.8.1
-      #   ONNXRT_PATH: onnxruntime==1.7.0
-      #   COREML_PATH: coremltools==4.0 # git+https://github.com/apple/coremltools@3.1
-      #   numpy.version: '<=1.23.5'
-
     maxParallel: 3
 
   steps:

diff --git a/onnxmltools/convert/lightgbm/_parse.py b/onnxmltools/convert/lightgbm/_parse.py
@@ -34,7 +34,10 @@ def __init__(self, booster):
         else:
             raise NotImplementedError(
                 'Unsupported LightGbm objective: %r.' % self.objective_)
-        average_output = self.booster_.attr('average_output')
+        try:
+            average_output = self.booster_.attr('average_output')
+        except AttributeError:
+            average_output = self.booster_.params.get("average_output", None)
         if average_output:
             self.boosting_type = 'rf'
         else:
@@ -47,7 +50,10 @@ def _generate_classes(booster):
         if isinstance(booster, dict):
             num_class = booster['num_class']
         else:
-            num_class = booster.attr('num_class')
+            try:
+                num_class = booster.attr('num_class')
+            except AttributeError:
+                num_class = booster.params.get('num_class', None)
         if num_class is None:
             dp = booster.dump_model(num_iteration=1)
             num_class = dp['num_class']
@@ -59,7 +65,10 @@ def get_objective(self):
         "Returns the objective."
         if hasattr(self, 'objective_') and self.objective_ is not None:
             return self.objective_
-        objective = self.booster_.attr('objective')
+        try:
+            objective = self.booster_.attr('objective')
+        except AttributeError:
+            objective = self.booster_.params.get("objective", None)
         if objective is not None:
             return objective
         dp = self.booster_.dump_model(num_iteration=1)

diff --git a/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py b/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py
@@ -264,20 +264,34 @@ def dump_booster_model(self, num_iteration=None, start_iteration=0,
     """
     if getattr(self, 'is_mock', False):
         return self.dump_model(), None
-    from lightgbm.basic import (
-        _LIB, FEATURE_IMPORTANCE_TYPE_MAPPER, _safe_call,
-        json_default_with_numpy)
+    from lightgbm.basic import _LIB, _safe_call
+    try:
+        # lightgbm >= 4.0
+        from lightgbm.basic import (
+            _FEATURE_IMPORTANCE_TYPE_MAPPER as FITM,
+            _json_default_with_numpy as jdwn,
+        )
+    except ImportError:
+        # lightgbm < 4.0
+        from lightgbm.basic import (
+            FEATURE_IMPORTANCE_TYPE_MAPPER as FITM,
+            json_default_with_numpy as jdwn,
+        )
     if num_iteration is None:
         num_iteration = self.best_iteration
-    importance_type_int = FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
+    importance_type_int = FITM[importance_type]
     buffer_len = 1 << 20
     tmp_out_len = ctypes.c_int64(0)
     string_buffer = ctypes.create_string_buffer(buffer_len)
     ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
     if verbose >= 2:
         print("[dump_booster_model] call CAPI: LGBM_BoosterDumpModel")
+    try:
+        handle = self._handle
+    except AttributeError:
+        handle = self.handle
     _safe_call(_LIB.LGBM_BoosterDumpModel(
-        self.handle,
+        handle,
         ctypes.c_int(start_iteration),
         ctypes.c_int(num_iteration),
         ctypes.c_int(importance_type_int),
@@ -290,8 +304,14 @@ def dump_booster_model(self, num_iteration=None, start_iteration=0,
         string_buffer = ctypes.create_string_buffer(actual_len)
         ptr_string_buffer = ctypes.c_char_p(
             *[ctypes.addressof(string_buffer)])
+        try:
+            # lightgbm >= 4.0
+            handle = self._handle
+        except AttributeError:
+            # lightgbm < 4.0
+            handle = self.handle
         _safe_call(_LIB.LGBM_BoosterDumpModel(
-            self.handle,
+            handle,
             ctypes.c_int(start_iteration),
             ctypes.c_int(num_iteration),
             ctypes.c_int(importance_type_int),
@@ -359,7 +379,7 @@ def hook(self, obj):
                      info=info, n_trees=self.num_trees(), verbose=verbose)
     ret['pandas_categorical'] = json.loads(
         json.dumps(self.pandas_categorical,
-                   default=json_default_with_numpy))
+                   default=jdwn))
     if verbose >= 2:
         print("[dump_booster_model] end.")
     return ret, info

diff --git a/onnxmltools/utils/tests_helper.py b/onnxmltools/utils/tests_helper.py
@@ -8,7 +8,7 @@
 from onnx.defs import onnx_opset_version
 from onnxconverter_common.onnx_ex import DEFAULT_OPSET_NUMBER
 from ..convert.common.data_types import FloatTensorType
-from .utils_backend import compare_backend, extract_options, evaluate_condition, is_backend_enabled
+from .utils_backend import compare_backend, extract_options, is_backend_enabled
 
 
 TARGET_OPSET = min(DEFAULT_OPSET_NUMBER, onnx_opset_version())
@@ -183,22 +183,9 @@ def dump_data_and_model(data, model, onnx=None, basename="model", folder=None,
             if not is_backend_enabled(b):
                 continue
             if isinstance(allow_failure, str):
-                allow = evaluate_condition(b, allow_failure)
-            else:
-                allow = allow_failure
-            if allow is None:
-                output = compare_backend(b, runtime_test, options=extract_options(basename),
-                                         context=context, verbose=verbose)
-            else:
-                try:
-                    output = compare_backend(b, runtime_test, options=extract_options(basename),
-                                             context=context, verbose=verbose)
-                except AssertionError as e:
-                    if isinstance(allow, bool) and allow:
-                        warnings.warn("Issue with '{0}' due to {1}".format(basename, e))
-                        continue
-                    else:
-                        raise e
+                raise NotImplementedError("allow_failure is deprecated.")
+            output = compare_backend(b, runtime_test, options=extract_options(basename),
+                                        context=context, verbose=verbose)
             if output is not None:
                 dest = os.path.join(folder, basename + ".backend.{0}.pkl".format(b))
                 names.append(dest)