merge conflicts

Signed-off-by: Xavier Dupre <xadupre@microsoft.com>

.azure-pipelines/linux-conda-CI.yml

@@ -123,11 +123,6 @@ jobs:
       pytest tests/sparkml --durations=0
     displayName: 'pytest - sparkml'
 
-  - script: |
-      export PYTHONPATH=.
-      pytest tests/svmlib --durations=0
-    displayName: 'pytest - svmlib'
-
   - script: |
       export PYTHONPATH=.
       pytest tests/utils --durations=0
@@ -144,6 +139,11 @@ jobs:
       pytest tests/h2o --durations=0
     displayName: 'pytest - h2o'
 
+  - script: |
+      export PYTHONPATH=.
+      pytest tests/svmlib --durations=0
+    displayName: 'pytest - svmlib'
+
   - script: |
       pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
       pip install hummingbird-ml --no-deps

onnxmltools/convert/lightgbm/_parse.py

@@ -42,7 +42,10 @@ def __init__(self, booster):
             raise NotImplementedError(
                 "Unsupported LightGbm objective: %r." % self.objective_
             )
-        average_output = self.booster_.attr("average_output")
+        try:
+            average_output = self.booster_.attr("average_output")
+        except AttributeError:
+            average_output = self.booster_.params.get("average_output", None)
         if average_output:
             self.boosting_type = "rf"
         else:
@@ -55,7 +58,10 @@ def _generate_classes(booster):
         if isinstance(booster, dict):
             num_class = booster["num_class"]
         else:
-            num_class = booster.attr("num_class")
+            try:
+                num_class = booster.attr("num_class")
+            except AttributeError:
+                num_class = booster.params.get("num_class", None)
         if num_class is None:
             dp = booster.dump_model(num_iteration=1)
             num_class = dp["num_class"]
@@ -67,7 +73,10 @@ def get_objective(self):
         "Returns the objective."
         if hasattr(self, "objective_") and self.objective_ is not None:
             return self.objective_
-        objective = self.booster_.attr("objective")
+        try:
+            objective = self.booster_.attr("objective")
+        except AttributeError:
+            objective = self.booster_.params.get("objective", None)
         if objective is not None:
             return objective
         dp = self.booster_.dump_model(num_iteration=1)

onnxmltools/convert/lightgbm/operator_converters/LightGbm.py

@@ -314,25 +314,36 @@ def dump_booster_model(
     """
     if getattr(self, "is_mock", False):
         return self.dump_model(), None
-    from lightgbm.basic import (
-        _LIB,
-        FEATURE_IMPORTANCE_TYPE_MAPPER,
-        _safe_call,
-        json_default_with_numpy,
-    )
+    from lightgbm.basic import _LIB, _safe_call
 
+    try:
+        # lightgbm >= 4.0
+        from lightgbm.basic import (
+            _FEATURE_IMPORTANCE_TYPE_MAPPER as FITM,
+            _json_default_with_numpy as jdwn,
+        )
+    except ImportError:
+        # lightgbm < 4.0
+        from lightgbm.basic import (
+            FEATURE_IMPORTANCE_TYPE_MAPPER as FITM,
+            json_default_with_numpy as jdwn,
+        )
     if num_iteration is None:
         num_iteration = self.best_iteration
-    importance_type_int = FEATURE_IMPORTANCE_TYPE_MAPPER[importance_type]
+    importance_type_int = FITM[importance_type]
     buffer_len = 1 << 20
     tmp_out_len = ctypes.c_int64(0)
     string_buffer = ctypes.create_string_buffer(buffer_len)
     ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
     if verbose >= 2:
         print("[dump_booster_model] call CAPI: LGBM_BoosterDumpModel")
+    try:
+        handle = self._handle
+    except AttributeError:
+        handle = self.handle
     _safe_call(
         _LIB.LGBM_BoosterDumpModel(
-            self.handle,
+            handle,
             ctypes.c_int(start_iteration),
             ctypes.c_int(num_iteration),
             ctypes.c_int(importance_type_int),
@@ -343,6 +354,28 @@ def dump_booster_model(
     )
     actual_len = tmp_out_len.value
     # if buffer length is not long enough, reallocate a buffer
+    if actual_len > buffer_len:
+        string_buffer = ctypes.create_string_buffer(actual_len)
+        ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
+        try:
+            # lightgbm >= 4.0
+            handle = self._handle
+        except AttributeError:
+            # lightgbm < 4.0
+            handle = self.handle
+        _safe_call(
+            _LIB.LGBM_BoosterDumpModel(
+                handle,
+                ctypes.c_int(start_iteration),
+                ctypes.c_int(num_iteration),
+                ctypes.c_int(importance_type_int),
+                ctypes.c_int64(buffer_len),
+                ctypes.byref(tmp_out_len),
+                ptr_string_buffer,
+            )
+        )
+    actual_len = tmp_out_len.value
+    # if buffer length is not long enough, reallocate a buffer
     if actual_len > buffer_len:
         string_buffer = ctypes.create_string_buffer(actual_len)
         ptr_string_buffer = ctypes.c_char_p(*[ctypes.addressof(string_buffer)])
@@ -424,7 +457,7 @@ def hook(self, obj):
         verbose=verbose,
     )
     ret["pandas_categorical"] = json.loads(
-        json.dumps(self.pandas_categorical, default=json_default_with_numpy)
+        json.dumps(self.pandas_categorical, default=jdwn)
     )
     if verbose >= 2:
         print("[dump_booster_model] end.")

onnxmltools/utils/tests_helper.py

@@ -2,17 +2,11 @@
 
 import pickle
 import os
-import warnings
 import numpy
 from onnx.defs import onnx_opset_version
 from onnxconverter_common.onnx_ex import DEFAULT_OPSET_NUMBER
 from ..convert.common.data_types import FloatTensorType
-from .utils_backend import (
-    compare_backend,
-    extract_options,
-    evaluate_condition,
-    is_backend_enabled,
-)
+from .utils_backend import compare_backend, extract_options, is_backend_enabled
 
 
 TARGET_OPSET = min(DEFAULT_OPSET_NUMBER, onnx_opset_version())
@@ -205,32 +199,14 @@ def dump_data_and_model(
             if not is_backend_enabled(b):
                 continue
             if isinstance(allow_failure, str):
-                allow = evaluate_condition(b, allow_failure)
-            else:
-                allow = allow_failure
-            if allow is None:
-                output = compare_backend(
-                    b,
-                    runtime_test,
-                    options=extract_options(basename),
-                    context=context,
-                    verbose=verbose,
-                )
-            else:
-                try:
-                    output = compare_backend(
-                        b,
-                        runtime_test,
-                        options=extract_options(basename),
-                        context=context,
-                        verbose=verbose,
-                    )
-                except AssertionError as e:
-                    if isinstance(allow, bool) and allow:
-                        warnings.warn("Issue with '{0}' due to {1}".format(basename, e))
-                        continue
-                    else:
-                        raise e
+                raise NotImplementedError("allow_failure is deprecated.")
+            output = compare_backend(
+                b,
+                runtime_test,
+                options=extract_options(basename),
+                context=context,
+                verbose=verbose,
+            )
             if output is not None:
                 dest = os.path.join(folder, basename + ".backend.{0}.pkl".format(b))
                 names.append(dest)

onnxmltools/utils/utils_backend.py

@@ -22,19 +22,6 @@ class OnnxRuntimeAssertionError(AssertionError):
     """
 
 
-def evaluate_condition(backend, condition):
-    """
-    Evaluates a condition such as
-    ``pv.Version(onnxruntime.__version__) <= pv.Version('0.1.3')``
-    """
-    if backend == "onnxruntime":
-        pass
-
-        return eval(condition)
-    else:
-        raise NotImplementedError("Not implemented for backend '{0}'".format(backend))
-
-
 def is_backend_enabled(backend):
     """
     Tells if a backend is enabled.

tests/coreml/test_cml_DictVectorizerConverter.py

@@ -54,11 +54,7 @@ def test_dict_vectorizer(self):
         model_onnx = convert(model_coreml.get_spec(), target_opset=TARGET_OPSET)
         self.assertTrue(model_onnx is not None)
         dump_data_and_model(
-            data,
-            model,
-            model_onnx,
-            basename="CmlDictVectorizer-OneOff-SkipDim1",
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.3.0')",
+            data, model, model_onnx, basename="CmlDictVectorizer-OneOff-SkipDim1"
         )
 
 

tests/coreml/test_cml_GLMClassifierConverter.py

@@ -55,11 +55,7 @@ def test_glm_classifier(self):
         self.assertTrue(lr_onnx is not None)
         self.validate_zipmap(lr_onnx)
         dump_data_and_model(
-            X.astype(numpy.float32),
-            lr,
-            lr_onnx,
-            basename="CmlbinLogitisticRegression",
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.3.0')",
+            X.astype(numpy.float32), lr, lr_onnx, basename="CmlbinLogitisticRegression"
         )
 
         # Ensure there is a probability output
@@ -70,11 +66,7 @@ def test_glm_classifier(self):
         self.assertTrue(svm_onnx is not None)
         self.validate_zipmap(svm_onnx)
         dump_data_and_model(
-            X.astype(numpy.float32),
-            svm,
-            svm_onnx,
-            basename="CmlBinLinearSVC-NoProb",
-            allow_failure=True,
+            X.astype(numpy.float32), svm, svm_onnx, basename="CmlBinLinearSVC-NoProb"
         )
 
 

tests/coreml/test_cml_SupportVectorClassifierConverter.py

@@ -78,9 +78,7 @@ def test_support_vector_classifier_binary_no_prob(self):
         nodes = svm_onnx.graph.node
         self.assertEqual(len(nodes), 1)
         self._check_model_outputs(svm_onnx, ["classLabel"])
-        dump_data_and_model(
-            X, svm, svm_onnx, basename="CmlBinSVC-Out0", allow_failure=True
-        )
+        dump_data_and_model(X, svm, svm_onnx, basename="CmlBinSVC-Out0")
 
     @unittest.skipIf(
         pv.Version(coremltools.__version__) > pv.Version("3.1"), reason="untested"

tests/coreml/test_cml_TreeEnsembleClassifierConverter.py

@@ -48,11 +48,7 @@ def test_tree_ensemble_classifier(self):
         self.assertTrue(model_onnx is not None)
         self.validate_zipmap(model_onnx)
         dump_data_and_model(
-            X,
-            model,
-            model_onnx,
-            basename="CmlBinRandomForestClassifier",
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.3.0')",
+            X, model, model_onnx, basename="CmlBinRandomForestClassifier"
         )
 
 

tests/coreml/test_cml_TreeEnsembleRegressorConverterXGBoost.py

@@ -54,7 +54,6 @@ def test_tree_ensemble_regressor_xgboost(self):
                 model,
                 model_onnx,
                 basename="CmlXGBoostRegressor-OneOff-Reshape",
-                allow_failure=True,
             )
 
 

tests/hummingbirdml/test_LightGbmTreeEnsembleConverters_hummingbird.py

@@ -60,7 +60,6 @@ def test_lightgbm_booster_classifier(self):
             X,
             model,
             model_onnx,
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.3.0')",
             basename=prefix + "BoosterBin" + model.__class__.__name__,
         )
 
@@ -109,7 +108,6 @@ def test_lightgbm_booster_classifier_zipmap(self):
             X,
             model,
             model_onnx,
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.3.0')",
             basename=prefix + "BoosterBin" + model.__class__.__name__,
         )
 
@@ -142,10 +140,11 @@ def test_lightgbm_booster_multi_classifier(self):
             X,
             model,
             model_onnx,
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.3.0')",
             basename=prefix + "BoosterBin" + model.__class__.__name__,
         )
-        sess = InferenceSession(model_onnx.SerializeToString())
+        sess = InferenceSession(
+            model_onnx.SerializeToString(), providers=["CPUExecutionProvider"]
+        )
         out = sess.get_outputs()
         names = [o.name for o in out]
         assert names == ["label", "probabilities"]
@@ -178,7 +177,6 @@ def test_lightgbm_booster_regressor(self):
             X,
             model,
             model_onnx,
-            allow_failure="pv.Version(onnx.__version__) < pv.Version('1.0.0')",
             basename=prefix + "BoosterBin" + model.__class__.__name__,
         )