From b9a68228c1f6b2a9c6dc3184a7a3a8f664f3d2e2 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Wed, 8 Jun 2022 15:18:25 -0400 Subject: [PATCH 1/5] fix enable messages --- stix2elevator/options.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/stix2elevator/options.py b/stix2elevator/options.py index a69e07eb..1f51ac97 100644 --- a/stix2elevator/options.py +++ b/stix2elevator/options.py @@ -221,8 +221,9 @@ def remove_silent(item, elements): if disabled: self._disabled = _convert_to_int_list(disabled) self._disabled = [x for x in self._disabled if x in CHECK_CODES] - for x in self._disabled: - remove_silent(x, self._enabled) + if hasattr(self, "__enabled"): + for x in self._disabled: + remove_silent(x, self._enabled) else: self._disabled = [] @@ -243,8 +244,9 @@ def remove_silent(item, elements): if enabled: self._enabled = _convert_to_int_list(enabled) self._enabled = [x for x in self._enabled if x in CHECK_CODES] - for x in self._enabled: - remove_silent(x, self._disabled) + if hasattr(self, "__disabled"): + for x in self._enabled: + remove_silent(x, self._disabled) else: self._enabled = copy.deepcopy(CHECK_CODES) From bc428ed309602c24950ac5fc0df5e1eedac4bee8 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Thu, 9 Jun 2022 17:00:41 -0400 Subject: [PATCH 2/5] exclusivity of --disable and --enable --- docs/command-line.rst | 10 ++++++--- docs/warnings.rst | 1 + stix2elevator/cli.py | 15 +++++++------- stix2elevator/options.py | 36 ++++++++++----------------------- stix2elevator/test/test_main.py | 33 +++++++++++++++++++++++++++++- 5 files changed, 59 insertions(+), 36 deletions(-) diff --git a/docs/command-line.rst b/docs/command-line.rst index 16215696..e19e5a70 100644 --- a/docs/command-line.rst +++ b/docs/command-line.rst @@ -84,14 +84,13 @@ optional arguments: -e ENABLED, --enable ENABLED A comma-separated list of the stix2-elevator messages - to enable. If the --disable option is not used, no - other messages will be shown. + to enable. Not to be used with --disable. Example: --enable 250 -d DISABLED, --disable DISABLED A comma-separated list of the stix2-elevator messages - to disable. + to disable. Not to be used with --enable. Example: --disable 212,220 @@ -136,4 +135,9 @@ Refer to the :ref:`warning_messages` section for all stix2-elevator messages. Us associated code number to ``--enable`` or ``--disable`` a message. By default, the stix2-elevator displays all messages. +The --enable and --disable arguments cannot be used at the same time. When a message code is not specified in the --enable +option it will not be displayed. When a message code is not specified in the --disable +option it will be displayed. If the number of messages codes to be both enable and disable are both large, it is sufficient +to just specify the shorter one. + Note: disabling the message does not disable any functionality. diff --git a/docs/warnings.rst b/docs/warnings.rst index 3f2fcab3..c01cff34 100644 --- a/docs/warnings.rst +++ b/docs/warnings.rst @@ -29,6 +29,7 @@ custom_property_prefix is provided, but the missing policy is not 'use-custom-pr Custom properties/objects/extensions are deprecated in version 2.1. Suggest using 'use-extensions' instead 215 info The missing policy option of 'use-extensions' cannot be used with version 2.0. 'use-custom-properies' is suggested 216 error ACS data markings cannot be supported in version 2.0. --acs option is ignored. 217 warn +Only one of the options --enable and --disable can be used 218 error ================================================================================================================== ==== ===== diff --git a/stix2elevator/cli.py b/stix2elevator/cli.py index beb1f964..64caabc4 100644 --- a/stix2elevator/cli.py +++ b/stix2elevator/cli.py @@ -115,8 +115,7 @@ def _get_arg_parser(is_script=True): "-e", "--enable", help="A comma-separated list of the stix2-elevator messages to enable. " - "If the --disable option is not used, no other messages will be " - "shown. \n\nExample: stix2_elevator.py --enable 250", + "Not to be used with --disable. \n\nExample: stix2_elevator.py --enable 250", dest="enabled", default=None ) @@ -125,7 +124,7 @@ def _get_arg_parser(is_script=True): "-d", "--disable", help="A comma-separated list of the stix2-elevator messages to disable. \n\n" - "Example: stix2_elevator.py --disable 212,220", + "Not to be used with --enable. \n\nExample: stix2_elevator.py --disable 212,220", dest="disabled", default=None ) @@ -208,10 +207,12 @@ def main(): elevator_parser = _get_arg_parser() elevator_args = elevator_parser.parse_args() sys.setrecursionlimit(3000) - initialize_options(options=elevator_args) - result = elevate(elevator_args.file_) - if result: - sys.stdout.write(result + "\n") + if initialize_options(options=elevator_args): + result = elevate(elevator_args.file_) + if result: + sys.stdout.write(result + "\n") + else: + sys.exit(1) else: sys.exit(1) diff --git a/stix2elevator/options.py b/stix2elevator/options.py index 1f51ac97..97073f38 100644 --- a/stix2elevator/options.py +++ b/stix2elevator/options.py @@ -3,6 +3,7 @@ import logging import os import shlex +import sys # external from stix2validator.scripts import stix2_validator @@ -210,20 +211,9 @@ def disabled(self): @disabled.setter def disabled(self, disabled): - def remove_silent(item, elements): - try: - elements.remove(item) - except ValueError: - pass # suppress exception if value is not present - # Convert string of comma-separated checks to a list, - # and convert check code numbers to names. By default no messages are - # disabled. if disabled: self._disabled = _convert_to_int_list(disabled) self._disabled = [x for x in self._disabled if x in CHECK_CODES] - if hasattr(self, "__enabled"): - for x in self._disabled: - remove_silent(x, self._enabled) else: self._disabled = [] @@ -233,22 +223,11 @@ def enabled(self): @enabled.setter def enabled(self, enabled): - def remove_silent(item, elements): - try: - elements.remove(item) - except ValueError: - pass # suppress exception if value is not present - # Convert string of comma-separated checks to a list, - # and convert check code numbers to names. By default all messages are - # enabled. if enabled: self._enabled = _convert_to_int_list(enabled) self._enabled = [x for x in self._enabled if x in CHECK_CODES] - if hasattr(self, "__disabled"): - for x in self._enabled: - remove_silent(x, self._disabled) else: - self._enabled = copy.deepcopy(CHECK_CODES) + self._enabled = [] def initialize_options(options=None): @@ -261,6 +240,10 @@ def initialize_options(options=None): else: ALL_OPTIONS = ElevatorOptions(options) + if ALL_OPTIONS.disabled and ALL_OPTIONS.enabled: + error("Only one of the options --enable and --disable can be used", 218) + return False + if ALL_OPTIONS.silent and ALL_OPTIONS.message_log_directory: info("Both console and output log have disabled messages.", 209) @@ -286,6 +269,7 @@ def initialize_options(options=None): if ALL_OPTIONS.acs and ALL_OPTIONS.spec_version == "2.0": warn("ACS data markings cannot be supported in version 2.0. --acs option is ignored.", 217) ALL_OPTIONS.acs = False + return True def get_validator_options(): @@ -313,10 +297,12 @@ def msg_id_enabled(msg_id): if get_option_value("silent"): return False - if not get_option_value("disabled"): + if get_option_value("disabled"): + return not (msg_id in get_option_value("disabled")) + elif get_option_value("enabled"): return msg_id in get_option_value("enabled") else: - return not (msg_id in get_option_value("disabled")) + return True # These codes are aligned with elevator_log_messages spreadsheet. diff --git a/stix2elevator/test/test_main.py b/stix2elevator/test/test_main.py index a91ec743..73ce450f 100644 --- a/stix2elevator/test/test_main.py +++ b/stix2elevator/test/test_main.py @@ -41,7 +41,7 @@ def setup_options(): message_log_directory=None, output_directory=None, markings_allowed="", acs=False, ignore_required_properties=False), ]) -def test_setup_options(opts): +def test_setup_options_with_disabled(opts): options.ALL_OPTIONS = None # To make sure we can set it again initialize_options(opts) assert get_option_value("policy") == "no_policy" @@ -50,6 +50,37 @@ def test_setup_options(opts): assert get_option_value("disabled") == [212, 901] +@pytest.mark.parametrize("opts", [ + ElevatorOptions(policy="no_policy", spec_version=get_environment_variable_value('VERSION'), log_level="DEBUG", enabled=[212, 901]), + {"policy": "no_policy", "spec_version": get_environment_variable_value('VERSION'), "log_level": "DEBUG", "enabled": [212, 901]}, + Namespace(policy="no_policy", spec_version=get_environment_variable_value('VERSION'), log_level="DEBUG", enabled="212,901", + file_=None, incidents=False, missing_policy=get_environment_variable_value("MISSING_POLICY"), + custom_property_prefix="elevator", infrastructure=False, package_created_by_id=None, + default_timestamp=None, validator_args="--strict-types", disabled=None, silent=False, + message_log_directory=None, output_directory=None, markings_allowed="", acs=False, + ignore_required_properties=False), +]) +def test_setup_options_with_enabled(opts): + options.ALL_OPTIONS = None # To make sure we can set it again + initialize_options(opts) + assert get_option_value("policy") == "no_policy" + assert get_option_value("spec_version") == get_environment_variable_value('VERSION') + assert get_option_value("log_level") == "DEBUG" + assert get_option_value("enabled") == [212, 901] + +@pytest.mark.parametrize("opts", [ + Namespace(policy="no_policy", spec_version=get_environment_variable_value('VERSION'), log_level="DEBUG", enabled="212,901", + file_=None, incidents=False, missing_policy=get_environment_variable_value("MISSING_POLICY"), + custom_property_prefix="elevator", infrastructure=False, package_created_by_id=None, + default_timestamp=None, validator_args="--strict-types", disabled="902", silent=False, + message_log_directory=None, output_directory=None, markings_allowed="", acs=False, + ignore_required_properties=False), +]) +def test_setup_options_with_enabled_and_disabled(opts): + options.ALL_OPTIONS = None # To make sure we can set it again + assert not initialize_options(opts) + + def test_elevate_with_marking_container(): setup_options() From c4fe123943c526a4a967aab6f9ea8bdae6f68609 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 10 Jun 2022 11:21:23 -0400 Subject: [PATCH 3/5] clean up some text --- docs/command-line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/command-line.rst b/docs/command-line.rst index e19e5a70..ef88229b 100644 --- a/docs/command-line.rst +++ b/docs/command-line.rst @@ -137,7 +137,7 @@ stix2-elevator displays all messages. The --enable and --disable arguments cannot be used at the same time. When a message code is not specified in the --enable option it will not be displayed. When a message code is not specified in the --disable -option it will be displayed. If the number of messages codes to be both enable and disable are both large, it is sufficient +option it will be displayed. If the number of messages codes to be both enabled and disabled are both large, it is sufficient to just specify the shorter one. Note: disabling the message does not disable any functionality. From f02c40db8e85c625d1dc39cbd9bd5396929a88bf Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 10 Jun 2022 11:30:32 -0400 Subject: [PATCH 4/5] flaky --- stix2elevator/convert_pattern.py | 12 ++++++------ stix2elevator/convert_stix.py | 25 ++++++++++++------------- stix2elevator/ids.py | 8 +++++--- stix2elevator/options.py | 2 -- stix2elevator/test/test_idioms.py | 1 + stix2elevator/test/test_main.py | 1 + stix2elevator/utils.py | 5 +++-- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/stix2elevator/convert_pattern.py b/stix2elevator/convert_pattern.py index e86e825a..dcc3998e 100644 --- a/stix2elevator/convert_pattern.py +++ b/stix2elevator/convert_pattern.py @@ -51,8 +51,7 @@ from stix2elevator.convert_cybox import split_into_requests_and_responses from stix2elevator.ids import ( add_id_value, exists_id_of_obs_in_characterizations, - exists_id_of_obs_in_sightings, exists_object_id_key, - get_id_value + exists_id_of_obs_in_sightings, exists_object_id_key, get_id_value ) from stix2elevator.missing_policy import ( check_for_missing_policy, convert_to_custom_name, @@ -60,7 +59,8 @@ ) from stix2elevator.options import error, get_option_value, info, warn from stix2elevator.utils import ( - encode_in_base64, find_key_in_dict_case_insensitive, identifying_info, map_vocabs_to_label + encode_in_base64, find_key_in_dict_case_insensitive, identifying_info, + map_vocabs_to_label ) from stix2elevator.vocab_mappings import WINDOWS_PEBINARY @@ -2772,9 +2772,9 @@ def remove_pattern_objects(bundle_instance): if obj["type"] == "sighting" and "observed_data_refs" in obj: for ref in obj["observed_data_refs"]: if exists_id_of_obs_in_sightings(ref) and ref in all_new_ids_with_patterns: - warn("Observable object from pattern cannot be an observed_data_ref of a sighting. See %s", - 644, - obj["id"]) + warn("Observable object from pattern cannot be an observed_data_ref of a sighting. See %s", + 644, + obj["id"]) new_remaining_objects = [] for obj in remaining_objects: if obj["type"] == "relationship" and "source_ref" in obj and "target_ref" in obj: diff --git a/stix2elevator/convert_stix.py b/stix2elevator/convert_stix.py index e51aa445..2f997d81 100644 --- a/stix2elevator/convert_stix.py +++ b/stix2elevator/convert_stix.py @@ -1,10 +1,10 @@ # Standard Library from datetime import datetime +from operator import attrgetter # external from cybox.core import Observable from lxml import etree -from operator import attrgetter import pycountry import stix from stix.campaign import Campaign @@ -67,9 +67,10 @@ ) from stix2elevator.convert_to_acs import convert_edh_marking_to_acs_marking from stix2elevator.ids import ( - add_id_of_obs_in_characterizations, add_id_of_obs_in_sightings, add_id_value, add_object_id_value, - exists_id_key, exists_ids_with_no_1x_object, generate_stix2x_id, - get_id_value, get_id_values, get_type_from_id, is_stix1x_id, record_ids + add_id_of_obs_in_characterizations, add_id_of_obs_in_sightings, + add_id_value, add_object_id_value, exists_id_key, + exists_ids_with_no_1x_object, generate_stix2x_id, get_id_value, + get_id_values, get_type_from_id, is_stix1x_id, record_ids ) from stix2elevator.missing_policy import ( check_for_missing_policy, convert_to_custom_name, @@ -83,10 +84,10 @@ add_label, add_marking_map_entry, apply_ais_markings, check_map_1x_markings_to_2x, convert_controlled_vocabs_to_open_vocabs, convert_timestamp_of_stix_object, convert_timestamp_to_string, - convert_to_stix_literal, find_key_in_dict_case_insensitive, identifying_info, iterpath, - lookup_marking_reference, map_1x_markings_to_2x, map_vocabs_to_label, - operation_on_path, set_tlp_reference, - strftime_with_appropriate_fractional_seconds + convert_to_stix_literal, find_key_in_dict_case_insensitive, + identifying_info, iterpath, lookup_marking_reference, + map_1x_markings_to_2x, map_vocabs_to_label, operation_on_path, + set_tlp_reference, strftime_with_appropriate_fractional_seconds ) from stix2elevator.vocab_mappings import ( ATTACK_MOTIVATION_MAP, COA_LABEL_MAP, INCIDENT_LABEL_MAP, @@ -359,10 +360,8 @@ def create_basic_object(stix2x_type, stix1x_obj, env, parent_id=None, id_used=Fa instance = {"type": stix2x_type} if get_option_value("spec_version") == "2.1": instance["spec_version"] = "2.1" - instance["id"] = generate_stix2x_id(stix2x_type, stix1x_obj.id_ if (stix1x_obj and - hasattr(stix1x_obj, "id_") and - stix1x_obj.id_) \ - else parent_id, + instance["id"] = generate_stix2x_id(stix2x_type, + stix1x_obj.id_ if (stix1x_obj and hasattr(stix1x_obj, "id_") and stix1x_obj.id_) else parent_id, id_used) if stix1x_obj: timestamp = convert_timestamp_of_stix_object(stix1x_obj, env.timestamp, True) @@ -2268,7 +2267,7 @@ def process_ttp_properties(sdo_instance, ttp, env, kill_chains_in_sdo=True, mark if ttp.exploit_targets is not None: warn("Exploit targets are part of STIX 1x %s. Assuming they are related.", 646, - "TTP" + (" " + ttp.id_ if hasattr(ttp,"id_") else "")) + "TTP" + (" " + ttp.id_ if hasattr(ttp, "id_") else "")) handle_relationship_to_refs(ttp.exploit_targets, sdo_instance["id"], env, "targets", marking_refs=marking_refs) if ttp.related_ttps: diff --git a/stix2elevator/ids.py b/stix2elevator/ids.py index 1c3d56a9..5a1aa016 100644 --- a/stix2elevator/ids.py +++ b/stix2elevator/ids.py @@ -10,9 +10,10 @@ # internal from stix2elevator.options import error, info, warn -from stix2elevator.utils import (find_key_in_dict_case_insensitive, - find_string_in_list_case_insensitive, - map_1x_type_to_20) +from stix2elevator.utils import ( + find_key_in_dict_case_insensitive, find_string_in_list_case_insensitive, + map_1x_type_to_20 +) def record_ids(stix_id, new_id): @@ -213,6 +214,7 @@ def add_object_id_value(key, value): if not value: warn("Can not associate %s with None", 611, key) + _ID_OF_OBSERVABLES_IN_SIGHTINGS = [] diff --git a/stix2elevator/options.py b/stix2elevator/options.py index 97073f38..b2a67b8c 100644 --- a/stix2elevator/options.py +++ b/stix2elevator/options.py @@ -1,9 +1,7 @@ # Standard Library -import copy import logging import os import shlex -import sys # external from stix2validator.scripts import stix2_validator diff --git a/stix2elevator/test/test_idioms.py b/stix2elevator/test/test_idioms.py index 7d63daae..d0a42686 100644 --- a/stix2elevator/test/test_idioms.py +++ b/stix2elevator/test/test_idioms.py @@ -141,6 +141,7 @@ def id_2x(id): except AttributeError: return False + def name_property(path): return "name" == path[0][-1] diff --git a/stix2elevator/test/test_main.py b/stix2elevator/test/test_main.py index 73ce450f..05c13c0e 100644 --- a/stix2elevator/test/test_main.py +++ b/stix2elevator/test/test_main.py @@ -68,6 +68,7 @@ def test_setup_options_with_enabled(opts): assert get_option_value("log_level") == "DEBUG" assert get_option_value("enabled") == [212, 901] + @pytest.mark.parametrize("opts", [ Namespace(policy="no_policy", spec_version=get_environment_variable_value('VERSION'), log_level="DEBUG", enabled="212,901", file_=None, incidents=False, missing_policy=get_environment_variable_value("MISSING_POLICY"), diff --git a/stix2elevator/utils.py b/stix2elevator/utils.py index 97a38fc7..3370aae2 100644 --- a/stix2elevator/utils.py +++ b/stix2elevator/utils.py @@ -13,10 +13,11 @@ # internal from stix2elevator.options import info, warn -def find_string_in_list_case_insensitive(item, l): + +def find_string_in_list_case_insensitive(item, ls): if item: item_as_lower = item.lower() - for s in l: + for s in ls: if item_as_lower == s.lower(): return True return False From a1e3b8eab668a99aa3d178b9fe0f1cac3aae6ea9 Mon Sep 17 00:00:00 2001 From: Rich Piazza Date: Fri, 10 Jun 2022 12:19:41 -0400 Subject: [PATCH 5/5] Bump version to 4.1.6 --- CHANGELOG | 4 ++++ README.rst | 2 +- docs/command-line.rst | 2 +- stix2elevator/version.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 82928caf..4ffac370 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,10 @@ CHANGELOG ========= +4.1.6 - 2022-06-10 + + - fix functionality for --enable and --disable options + 4.1.5 - 2022-02-04 - Additional fix for Hostname in SocketAddress in pattern diff --git a/README.rst b/README.rst index 5d34f3b0..06b89652 100644 --- a/README.rst +++ b/README.rst @@ -134,7 +134,7 @@ STIX 1.x content to STIX 2.x content: file -stix2-elevator v4.1.5 +stix2-elevator v4.1.6 positional arguments: diff --git a/docs/command-line.rst b/docs/command-line.rst index ef88229b..e2d0e1c0 100644 --- a/docs/command-line.rst +++ b/docs/command-line.rst @@ -27,7 +27,7 @@ STIX 1.x content to STIX 2.x content: file -stix2-elevator v4.1.5 +stix2-elevator v4.1.6 positional arguments: diff --git a/stix2elevator/version.py b/stix2elevator/version.py index 50a59288..c9051481 100644 --- a/stix2elevator/version.py +++ b/stix2elevator/version.py @@ -1 +1 @@ -__version__ = "4.1.5" +__version__ = "4.1.6"