From ae3eb7fb8f5e176d0a821adc15feeb78310f0197 Mon Sep 17 00:00:00 2001 From: richardyu-ms Date: Sat, 24 Dec 2022 12:42:50 +0000 Subject: [PATCH] [SAI-PTF] Skip test when hit expected error from sai api Why When hit the expected error like SAI_STATUS_NOT_SUPPORTED = -2, then skip the test. related to issue https://github.com/opencomputeproject/SAI/issues/1610 How Add conditional exeption handler when return from sai api invocation Test: Test on brcm platform on case sainexthop.tunnelVrfTest Signed-off-by: richardyu-ms --- meta/gensairpc.pl | 3 -- meta/rpc/README.md | 1 - meta/templates/sai_adapter.py.tt | 25 +++++++++++++-- meta/templates/sai_adapter_utils.tt | 48 ----------------------------- ptf/utest/TemplateTest.py | 2 +- 5 files changed, 23 insertions(+), 56 deletions(-) diff --git a/meta/gensairpc.pl b/meta/gensairpc.pl index b1a6df80d..0fde05fb5 100755 --- a/meta/gensairpc.pl +++ b/meta/gensairpc.pl @@ -90,7 +90,6 @@ [ 'verbose|v', 'Print more details', { default => 0 } ], [ 'mandatory-attrs', 'Make mandatory attributes obligatory in sai_adapter.py', { default => 0 } ], [ 'dev-utils:s', 'Generate additional development utils within the generated code. Additional options: [=log,zero]', { default => 0 } ], - [ 'skip_error:s', 'Skip test on specified error code. Additional options: [=-2]', { default => 0 } ], [ 'adapter_logger', 'Enable the logger in sai_adapter, it will log all the method invocation.', { default => 0} ], [ 'attr-header', 'Generate additional header of attributes definitions (including object types)', { default => 0 } ], [ 'help|h', 'Print this help', { shortcircuit => 1 } ], @@ -116,7 +115,6 @@ my $clean = $args->clean_meta; my $mandatory_attrs = $args->mandatory_attrs; my $dev_utils = ( $args->dev_utils ne q{} ? $args->dev_utils : 1 ); -my $skip_error = ( $args->skip_error ne q{} ? $args->skip_error : 1 ); my $adapter_logger = ( $args->adapter_logger ne q{} ? $args->adapter_logger : 1 ); my $attr_header = $args->attr_header; @@ -163,7 +161,6 @@ dbg => $dbg, mandatory_attrs => $mandatory_attrs, dev_utils => $dev_utils, - skip_error => $skip_error, adapter_logger => $adapter_logger, templates_dir => $templates_dir }; diff --git a/meta/rpc/README.md b/meta/rpc/README.md index 0fb8fb31b..8f75ea0e8 100644 --- a/meta/rpc/README.md +++ b/meta/rpc/README.md @@ -19,7 +19,6 @@ Optional arguments are useful for development purposes. | `-d` `--dump` | Dump all data to the file. **Should be used during development**. | | `--mandatory-attrs` | Make mandatory attributes obligatory in *sai\_adapter.py*. It removes `=None` from attributes, which are passed as arguments into python functions. Can be useful for debugging purposes, but since most of attributes are **optionally** mandatory, this is not as useful as it could be. | | `--dev-utils[=STR]` | Generate additional development utils within the generated code. Additional options: [=log,zero]. Useful for tests development and debugging. The generated code **should not** be committed. | -| `--skip_error[=STR]` | Skip test on specified error code. Additional options: [=-2]. The generated code **should not** be committed. | | `--adapter_logger` | Enable the logger in sai_adapter, it will log all the method invocation. | | `-h` `--help` | Print the help. | diff --git a/meta/templates/sai_adapter.py.tt b/meta/templates/sai_adapter.py.tt index 8ddcee7ff..5d96a6ab1 100644 --- a/meta/templates/sai_adapter.py.tt +++ b/meta/templates/sai_adapter.py.tt @@ -191,6 +191,12 @@ client.[% function.thrift_name %]( [%- BLOCK catch_exception -%] except sai_thrift_exception as e: status = e.status + if SKIP_TEST_ON_EXPECTED_ERROR and status in EXPECTED_ERROR_CODE: + reason = "SkipTest on expected error. [% function.thrift_name %] with errorcode: {} error: {}".format( + status, e) + print(reason) + testutils.skipped_test_count=1 + raise SkipTest(reason) if CATCH_EXCEPTIONS: [%- IF function.operation == 'stats' %] pass @@ -409,6 +415,12 @@ client.[% function.thrift_name %]( [%- BLOCK return_from_empty_function -%] global status status = SAI_STATUS_NOT_SUPPORTED + if SKIP_TEST_ON_EXPECTED_ERROR and status in EXPECTED_ERROR_CODE: + reason = "SkipTest on expected error. [% function.name %] with errorcode: {} error: {}".format( + status, e) + print(reason) + testutils.skipped_test_count=1 + raise SkipTest(reason) if CATCH_EXCEPTIONS: [%- IF function.operation == 'create' AND NOT function.rpc_return.is_list %] @@ -431,7 +443,6 @@ client.[% function.thrift_name %]( [%- PROCESS decorate_method IF dev_utils -%] - [%- PROCESS decorate_skip_test_on_error IF skip_error -%] [%- PROCESS decorate_invocation_logger IF adapter_logger -%] [%- PROCESS function_header %] [%- PROCESS function_docstring %] @@ -489,9 +500,11 @@ Thrift SAI interface basic tests # pylint: disable=too-many-return-statements,line-too-long,invalid-name [%- PROCESS dev_utils_imports IF dev_utils -%] -[%- PROCESS skip_test_on_error_imports IF skip_error -%] [%- PROCESS invocation_logger_imports IF adapter_logger -%] +from unittest import SkipTest +from ptf import testutils + from sai_thrift.ttypes import * from sai_thrift.sai_headers import * @@ -503,10 +516,16 @@ from sai_thrift.sai_headers import * # In order to catch exceptions and get error codes # in the application, it should be disabled. CATCH_EXCEPTIONS = True +# Expected error code +# Used with SKIP_TEST_ON_EXPECTED_ERROR +# For some expected errors in test +# Like SAI_STATUS_NOT_SUPPORTED = -2 +EXPECTED_ERROR_CODE = [-2] +# Skip test when hitting an expected error +SKIP_TEST_ON_EXPECTED_ERROR = True status = 0 [%- PROCESS dev_utils IF dev_utils -%] -[%- PROCESS skip_error IF skip_error -%] [%- PROCESS invocation_logger IF adapter_logger -%] [%- FOREACH api IN apis.keys.sort -%] diff --git a/meta/templates/sai_adapter_utils.tt b/meta/templates/sai_adapter_utils.tt index ec6690bf5..119d92bff 100644 --- a/meta/templates/sai_adapter_utils.tt +++ b/meta/templates/sai_adapter_utils.tt @@ -4,13 +4,7 @@ from __future__ import print_function from collections import defaultdict from collections import Counter -[% IF NOT skip_error %]from functools import wraps[% END %] -[% END -%] - -[%- BLOCK skip_test_on_error_imports %] from functools import wraps -from unittest import SkipTest -from ptf import testutils [% END -%] [%- BLOCK invocation_logger_imports %] @@ -29,13 +23,6 @@ import logging [% PROCESS instance_counter %] [% END -%] -[%- BLOCK skip_error %] - -# skip_error - -[% PROCESS skip_test_on_error %] -[% END -%] - [%- BLOCK invocation_logger %] # invocation_logger @@ -186,38 +173,6 @@ instance_counter.removed = defaultdict(Counter) instance_counter.not_removed = defaultdict(Counter) [%- END -%] -[%- BLOCK skip_test_on_error -%] -def skip_test_on_error(errorcode=[-2]): - def skip_test_on_error_decorator(func): - """ - Decorator for skip the test when error happened. - Args: - errorcode: a list of the error code that test will be skipped. - """ - - @wraps(func) - def decorated(*args, **kwargs): - """ - Check the return value and check if the status is the error code - on which the test should be skipped. - - Args: - args(List): original args - kwargs(Dict): original kwargs - Returns: - retval(Any): the original return value - """ - retval = func(*args, **kwargs) - global status - if status in errorcode: - reason = "SkipTest: {} with errorcode: {}".format(func.__name__, status) - print(reason) - testutils.skipped_test_count=1 - raise SkipTest(reason) - return retval - return decorated - return skip_test_on_error_decorator -[%- END -%] [%- BLOCK invocation_logger_func -%] def invocation_logger(func): @@ -259,9 +214,6 @@ def invocation_logger(func): @instance_counter("[% function.object %]", "[% function.operation %]"[% IF dev_utils.match('log') %], log=True[% END %][% IF dev_utils.match('zero') %], zero=True[% END %]) [%- END -%] -[%- BLOCK decorate_skip_test_on_error %] -@skip_test_on_error([% IF skip_error %]errorcode=[[% skip_error %]][% END %]) -[%- END -%] [%- BLOCK decorate_invocation_logger %] @invocation_logger diff --git a/ptf/utest/TemplateTest.py b/ptf/utest/TemplateTest.py index 969ca8f4a..b294ccadb 100644 --- a/ptf/utest/TemplateTest.py +++ b/ptf/utest/TemplateTest.py @@ -11,7 +11,7 @@ """ Class use to test the template. Run command under folder 'meta' - perl -Irpc gensairpc.pl --adapter_logger --skip_error=-2 + perl -Irpc gensairpc.pl --adapter_logger copy sai_adaptor to ./test/saithriftv2/gen-py/sai cd ./test/saithriftv2 sudo python3 setup.py install