From aa0c44b9f9c4cc6023530694ec3f02fa9da7135d Mon Sep 17 00:00:00 2001 From: Wenbing Li Date: Mon, 1 Jun 2020 10:59:42 -0700 Subject: [PATCH 1/4] Add the flake8 to be the default code formater. --- .azure-pipelines/linux-conda-CI.yml | 4 ++++ .azure-pipelines/win32-conda-CI.yml | 4 ++++ .flake8 | 10 ++++++++++ .../convert/common/_apply_operation.py | 2 +- onnxmltools/convert/common/_container.py | 2 ++ onnxmltools/convert/common/_registration.py | 2 +- onnxmltools/convert/common/_topology.py | 2 +- .../convert/common/case_insensitive_dict.py | 2 +- onnxmltools/convert/common/data_types.py | 2 +- onnxmltools/convert/common/interface.py | 2 +- onnxmltools/convert/common/optimizer.py | 7 ------- .../convert/common/shape_calculator.py | 2 +- onnxmltools/convert/common/tree_ensemble.py | 2 +- onnxmltools/convert/common/utils.py | 2 +- onnxmltools/convert/h2o/_parse.py | 2 +- onnxmltools/convert/h2o/convert.py | 5 +++-- onnxmltools/convert/lightgbm/_parse.py | 8 ++++---- onnxmltools/convert/lightgbm/convert.py | 5 ++--- .../lightgbm/operator_converters/LightGbm.py | 2 +- onnxmltools/convert/main.py | 2 +- onnxmltools/proto/__init__.py | 20 +++++++++---------- onnxmltools/utils/float16_converter.py | 2 +- onnxmltools/utils/main.py | 7 ++++--- requirements-dev.txt | 1 + 24 files changed, 57 insertions(+), 42 deletions(-) create mode 100644 .flake8 delete mode 100644 onnxmltools/convert/common/optimizer.py diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index 06094828..9ed8f75b 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -71,6 +71,10 @@ jobs: make lib displayName: 'Install dependencies' + - script: | + python -m flake8 ./onnxmltools + displayName: 'run flake8 check' + - script: | export PYTHONPATH=$PYTHONPATH:libsvm/python python -c "import svmutil" diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index 2dbac7df..7f235e62 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -87,6 +87,10 @@ jobs: python -m pip show pytest displayName: 'Install dependencies' + - script: | + python -m flake8 ./onnxmltools + displayName: 'run flake8 check' + - script: | call activate py$(python.version) set PYTHONPATH=libsvm\python;%PYTHONPATH% diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..98f34b13 --- /dev/null +++ b/.flake8 @@ -0,0 +1,10 @@ +[flake8] +max-line-length = 120 +per-file-ignores = + __init__.py:F401 +exclude = + sparkml + coreml + libsvm + xgboost + utils diff --git a/onnxmltools/convert/common/_apply_operation.py b/onnxmltools/convert/common/_apply_operation.py index 164488c1..a1373a2f 100644 --- a/onnxmltools/convert/common/_apply_operation.py +++ b/onnxmltools/convert/common/_apply_operation.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.onnx_ops import * +from onnxconverter_common.onnx_ops import * # noqa diff --git a/onnxmltools/convert/common/_container.py b/onnxmltools/convert/common/_container.py index 9915dbed..975d6e43 100644 --- a/onnxmltools/convert/common/_container.py +++ b/onnxmltools/convert/common/_container.py @@ -17,9 +17,11 @@ class LightGbmModelContainer(CommonSklearnModelContainer): class XGBoostModelContainer(CommonSklearnModelContainer): pass + class H2OModelContainer(CommonSklearnModelContainer): pass + class SparkmlModelContainer(RawModelContainer): def __init__(self, sparkml_model): diff --git a/onnxmltools/convert/common/_registration.py b/onnxmltools/convert/common/_registration.py index 7147e781..6a0317e5 100644 --- a/onnxmltools/convert/common/_registration.py +++ b/onnxmltools/convert/common/_registration.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.registration import * +from onnxconverter_common.registration import * # noqa diff --git a/onnxmltools/convert/common/_topology.py b/onnxmltools/convert/common/_topology.py index fe5054e3..f412f9ba 100644 --- a/onnxmltools/convert/common/_topology.py +++ b/onnxmltools/convert/common/_topology.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.topology import * +from onnxconverter_common.topology import * # noqa diff --git a/onnxmltools/convert/common/case_insensitive_dict.py b/onnxmltools/convert/common/case_insensitive_dict.py index 36986344..b566f73f 100644 --- a/onnxmltools/convert/common/case_insensitive_dict.py +++ b/onnxmltools/convert/common/case_insensitive_dict.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.case_insensitive_dict import * +from onnxconverter_common.case_insensitive_dict import * # noqa diff --git a/onnxmltools/convert/common/data_types.py b/onnxmltools/convert/common/data_types.py index 2aa57dbe..6603d1d4 100644 --- a/onnxmltools/convert/common/data_types.py +++ b/onnxmltools/convert/common/data_types.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.data_types import * +from onnxconverter_common.data_types import * # noqa diff --git a/onnxmltools/convert/common/interface.py b/onnxmltools/convert/common/interface.py index 7c8e3652..a892f1d6 100644 --- a/onnxmltools/convert/common/interface.py +++ b/onnxmltools/convert/common/interface.py @@ -6,4 +6,4 @@ # This file defines the interface of the converter internal object for callback, # So the usage of the methods and properties list here will not be affected among the different versions. -from onnxconverter_common.interface import * +from onnxconverter_common.interface import * # noqa diff --git a/onnxmltools/convert/common/optimizer.py b/onnxmltools/convert/common/optimizer.py deleted file mode 100644 index 3eb9b3f0..00000000 --- a/onnxmltools/convert/common/optimizer.py +++ /dev/null @@ -1,7 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from onnxconverter_common.optimizer import * diff --git a/onnxmltools/convert/common/shape_calculator.py b/onnxmltools/convert/common/shape_calculator.py index 021a410a..f1aa0439 100644 --- a/onnxmltools/convert/common/shape_calculator.py +++ b/onnxmltools/convert/common/shape_calculator.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.shape_calculator import * +from onnxconverter_common.shape_calculator import * # noqa diff --git a/onnxmltools/convert/common/tree_ensemble.py b/onnxmltools/convert/common/tree_ensemble.py index 55e520db..85f754a0 100644 --- a/onnxmltools/convert/common/tree_ensemble.py +++ b/onnxmltools/convert/common/tree_ensemble.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.tree_ensemble import * +from onnxconverter_common.tree_ensemble import * # noqa diff --git a/onnxmltools/convert/common/utils.py b/onnxmltools/convert/common/utils.py index 8ae9ae98..5428b31c 100644 --- a/onnxmltools/convert/common/utils.py +++ b/onnxmltools/convert/common/utils.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.utils import * +from onnxconverter_common.utils import * # noqa diff --git a/onnxmltools/convert/h2o/_parse.py b/onnxmltools/convert/h2o/_parse.py index 5635db96..4a67673d 100644 --- a/onnxmltools/convert/h2o/_parse.py +++ b/onnxmltools/convert/h2o/_parse.py @@ -7,6 +7,7 @@ from ..common._container import H2OModelContainer from ..common._topology import Topology + def _parse_h2o(scope, model, inputs): ''' :param scope: Scope object @@ -30,7 +31,6 @@ def _parse_h2o(scope, model, inputs): def parse_h2o(model, initial_types=None, target_opset=None, custom_conversion_functions=None, custom_shape_calculators=None): - raw_model_container = H2OModelContainer(model) topology = Topology(raw_model_container, default_batch_size='None', initial_types=initial_types, target_opset=target_opset, diff --git a/onnxmltools/convert/h2o/convert.py b/onnxmltools/convert/h2o/convert.py index 24ba9001..3d326bf7 100644 --- a/onnxmltools/convert/h2o/convert.py +++ b/onnxmltools/convert/h2o/convert.py @@ -16,7 +16,7 @@ from ._parse import parse_h2o # Invoke the registration of all our converters and shape calculators -from . import operator_converters, shape_calculators +from . import operator_converters, shape_calculators # noqa def convert(model, name=None, initial_types=None, doc_string='', target_opset=None, @@ -66,7 +66,8 @@ def convert(model, name=None, initial_types=None, doc_string='', target_opset=No mojo_str = h2o.print_mojo(model_path, format="json") mojo_model = json.loads(mojo_str) if mojo_model["params"]["algo"] != "gbm": - raise ValueError("Model type not supported (algo=%s). Only GBM Mojo supported for now." % mojo_model["params"]["algo"]) + raise ValueError( + "Model type not supported (algo=%s). Only GBM Mojo supported for now." % mojo_model["params"]["algo"]) target_opset = target_opset if target_opset else get_maximum_opset_supported() topology = parse_h2o(mojo_model, initial_types, target_opset, custom_conversion_functions, custom_shape_calculators) diff --git a/onnxmltools/convert/lightgbm/_parse.py b/onnxmltools/convert/lightgbm/_parse.py index e7e71f92..ef9369c2 100644 --- a/onnxmltools/convert/lightgbm/_parse.py +++ b/onnxmltools/convert/lightgbm/_parse.py @@ -8,7 +8,7 @@ from ..common._container import LightGbmModelContainer from ..common._topology import Topology from ..common.data_types import (FloatTensorType, - SequenceType, DictionaryType, StringType, Int64Type) + SequenceType, DictionaryType, StringType, Int64Type) from lightgbm import LGBMClassifier, LGBMRegressor @@ -19,6 +19,7 @@ lightgbm_operator_name_map = {LGBMClassifier: 'LgbmClassifier', LGBMRegressor: 'LgbmRegressor'} + class WrappedBooster: def __init__(self, booster): @@ -44,7 +45,7 @@ def __init__(self, booster): def _generate_classes(self, model_dict): if model_dict['num_class'] == 1: return numpy.asarray([0, 1]) - return numpy.arange(model_dict['num_class']) + return numpy.arange(model_dict['num_class']) def _get_lightgbm_operator_name(model): @@ -91,7 +92,7 @@ def _parse_lightgbm_simple_model(scope, model, inputs): def _parse_sklearn_classifier(scope, model, inputs): probability_tensor = _parse_lightgbm_simple_model( - scope, model, inputs) + scope, model, inputs) this_operator = scope.declare_local_operator('LgbmZipMap') this_operator.inputs = probability_tensor classes = model.classes_ @@ -145,7 +146,6 @@ def _parse_lightgbm(scope, model, inputs): def parse_lightgbm(model, initial_types=None, target_opset=None, custom_conversion_functions=None, custom_shape_calculators=None): - raw_model_container = LightGbmModelContainer(model) topology = Topology(raw_model_container, default_batch_size='None', initial_types=initial_types, target_opset=target_opset, diff --git a/onnxmltools/convert/lightgbm/convert.py b/onnxmltools/convert/lightgbm/convert.py index 39956be3..8a9f702b 100644 --- a/onnxmltools/convert/lightgbm/convert.py +++ b/onnxmltools/convert/lightgbm/convert.py @@ -12,8 +12,7 @@ from ._parse import parse_lightgbm, WrappedBooster # Invoke the registration of all our converters and shape calculators -# from . import shape_calculators -from . import operator_converters, shape_calculators +from . import operator_converters, shape_calculators # noqa def convert(model, name=None, initial_types=None, doc_string='', target_opset=None, @@ -22,7 +21,7 @@ def convert(model, name=None, initial_types=None, doc_string='', target_opset=No ''' This function produces an equivalent ONNX model of the given lightgbm model. The supported lightgbm modules are listed below. - + * `LGBMClassifiers `_ * `LGBMRegressor `_ * `Booster `_ diff --git a/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py b/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py index 17e2eb27..08e43204 100644 --- a/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py +++ b/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py @@ -8,7 +8,7 @@ import numbers import numpy as np from collections import Counter -from ...common._apply_operation import apply_div, apply_reshape, apply_sub, apply_cast +from ...common._apply_operation import apply_div, apply_reshape, apply_sub, apply_cast, apply_identity from ...common._registration import register_converter from ...common.tree_ensemble import get_default_tree_classifier_attribute_pairs from ....proto import onnx_proto diff --git a/onnxmltools/convert/main.py b/onnxmltools/convert/main.py index da18b1fd..81d29699 100644 --- a/onnxmltools/convert/main.py +++ b/onnxmltools/convert/main.py @@ -168,7 +168,7 @@ def convert_tensorflow(frozen_graph_def, debug_mode=False, custom_op_conversions=None): try: importlib.import_module('tf2onnx') - except (ImportError, ModuleNotFoundError) as e: + except (ImportError, ModuleNotFoundError): raise RuntimeError('tf2onnx is not installed, please install it before calling this function.') return _convert_tf_wrapper(frozen_graph_def, name, input_names, output_names, doc_string, diff --git a/onnxmltools/proto/__init__.py b/onnxmltools/proto/__init__.py index bd9feca0..896f0d64 100644 --- a/onnxmltools/proto/__init__.py +++ b/onnxmltools/proto/__init__.py @@ -3,6 +3,16 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +# Rather than using ONNX protobuf definition throughout our codebase, we import ONNX protobuf definition here so that +# we can conduct quick fixes by overwriting ONNX functions without changing any lines elsewhere. +from onnx import onnx_pb as onnx_proto # noqa +from onnx import helper + +# Overwrite the make_tensor defined in onnx.helper because of a bug (string tensor get assigned twice) +from onnx import mapping +from onnx.onnx_pb import TensorProto +from onnx.helper import split_complex_to_pairs + def _check_onnx_version(): import pkg_resources @@ -13,16 +23,6 @@ def _check_onnx_version(): _check_onnx_version() -# Rather than using ONNX protobuf definition throughout our codebase, we import ONNX protobuf definition here so that -# we can conduct quick fixes by overwriting ONNX functions without changing any lines elsewhere. -from onnx import onnx_pb as onnx_proto -from onnx import helper - -# Overwrite the make_tensor defined in onnx.helper because of a bug (string tensor get assigned twice) -from onnx import mapping -from onnx.onnx_pb import TensorProto -from onnx.helper import split_complex_to_pairs - def _make_tensor_fixed(name, data_type, dims, vals, raw=False): ''' diff --git a/onnxmltools/utils/float16_converter.py b/onnxmltools/utils/float16_converter.py index 11203ea7..358e36fa 100644 --- a/onnxmltools/utils/float16_converter.py +++ b/onnxmltools/utils/float16_converter.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -from onnxconverter_common.float16 import * +from onnxconverter_common.float16 import * # noqa diff --git a/onnxmltools/utils/main.py b/onnxmltools/utils/main.py index 7f86bf70..e6fc6d55 100644 --- a/onnxmltools/utils/main.py +++ b/onnxmltools/utils/main.py @@ -1,8 +1,8 @@ -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. -#-------------------------------------------------------------------------- +# -------------------------------------------------------------------------- import os from ..proto import onnx_proto @@ -116,5 +116,6 @@ def set_model_doc_string(model, doc, override=False): if not convert_utils.is_string_type(doc): raise ValueError("Doc must be a string type.") if model.doc_string and not doc and override is False: - raise ValueError("Failing to overwrite the doc string with a blank string, set override to True if intentional.") + raise ValueError( + "Failing to overwrite the doc string with a blank string, set override to True if intentional.") model.doc_string = doc diff --git a/requirements-dev.txt b/requirements-dev.txt index a8b53aa8..956af62c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ scipy svm wheel xgboost +flake8 From 8eed29b619744cf32eb53040e18e5f55144aa3f5 Mon Sep 17 00:00:00 2001 From: Wenbing Li <10278425+wenbingl@users.noreply.github.com> Date: Mon, 1 Jun 2020 11:32:06 -0700 Subject: [PATCH 2/4] Update win32-conda-CI.yml --- .azure-pipelines/win32-conda-CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index 7f235e62..c84ddd74 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -88,6 +88,7 @@ jobs: displayName: 'Install dependencies' - script: | + call activate py$(python.version) python -m flake8 ./onnxmltools displayName: 'run flake8 check' From 45b34884c40fec8390aedc5e57009c1c3a18e7c7 Mon Sep 17 00:00:00 2001 From: Wenbing Li Date: Mon, 1 Jun 2020 16:04:03 -0700 Subject: [PATCH 3/4] compatible with python 3.5 --- onnxmltools/convert/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/onnxmltools/convert/main.py b/onnxmltools/convert/main.py index 81d29699..214c8552 100644 --- a/onnxmltools/convert/main.py +++ b/onnxmltools/convert/main.py @@ -166,9 +166,8 @@ def convert_tensorflow(frozen_graph_def, target_opset=None, channel_first_inputs=None, debug_mode=False, custom_op_conversions=None): - try: - importlib.import_module('tf2onnx') - except (ImportError, ModuleNotFoundError): + import pkgutil + if not pkgutil.find_loader('tf2onnx'): raise RuntimeError('tf2onnx is not installed, please install it before calling this function.') return _convert_tf_wrapper(frozen_graph_def, name, input_names, output_names, doc_string, From b87dc5f2252b0a82c4cb5217c5782ff21d0227bd Mon Sep 17 00:00:00 2001 From: Wenbing Li Date: Mon, 1 Jun 2020 17:05:14 -0700 Subject: [PATCH 4/4] ok, flake8 now rejects bad lines. --- onnxmltools/convert/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onnxmltools/convert/main.py b/onnxmltools/convert/main.py index 214c8552..21011443 100644 --- a/onnxmltools/convert/main.py +++ b/onnxmltools/convert/main.py @@ -7,7 +7,6 @@ import onnx from .common import utils import warnings -import importlib def convert_coreml(model, name=None, initial_types=None, doc_string='', target_opset=None,