fix dictionary list, add some doc

stix2/datastore/relational_db/database_backends/database_backend_base.py

@@ -14,13 +14,22 @@ def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any)
         self.database_exists = database_exists(database_connection_url)
 
         if force_recreate:
-            if self.database_exists:
-                drop_database(database_connection_url)
-            create_database(database_connection_url)
-            self.database_exists = database_exists(database_connection_url)
+            self._create_database()
 
         self.database_connection = create_engine(database_connection_url)
 
+    def _create_database(self):
+        if self.database_exists:
+            drop_database(self.database_connection_url)
+        create_database(self.database_connection_url)
+        self.database_exists = database_exists(self.database_connection_url)
+
+    # =========================================================================
+    # schema methods
+
+    # the base methods assume schemas are not supported for the database
+    # ---------------------------------------------------------------------------
+
     def _create_schemas(self):
         pass
 
@@ -29,29 +38,17 @@ def determine_schema_name(stix_object):
         return ""
 
     @staticmethod
-    def determine_stix_type(stix_object):
-        if isinstance(stix_object, _DomainObject):
-            return "sdo"
-        elif isinstance(stix_object, _Observable):
-            return "sco"
-        elif isinstance(stix_object, _RelationshipObject):
-            return "sro"
-        elif isinstance(stix_object, _MetaObject):
-            return "common"
-
-    def _create_database(self):
-        if self.database_exists:
-            drop_database(self.database_connection.url)
-        create_database(self.database_connection.url)
-        self.database_exists = database_exists(self.database_connection.url)
-
     def schema_for(stix_class):
         return ""
 
     @staticmethod
     def schema_for_core():
         return ""
 
+    # =========================================================================
+    # sql type methods
+
+    # Database specific SQL types for STIX property classes
     # you must implement the next 4 methods in the subclass
 
     @staticmethod
@@ -70,6 +67,9 @@ def determine_sql_type_for_hex_property():  # noqa: F811
     def determine_sql_type_for_timestamp_property():  # noqa: F811
         pass
 
+    # ------------------------------------------------------------------
+    # Common SQL types for STIX property classes
+
     @staticmethod
     def determine_sql_type_for_kill_chain_phase():  # noqa: F811
         return None
@@ -102,11 +102,25 @@ def determine_sql_type_for_key_as_int():  # noqa: F811
     def determine_sql_type_for_key_as_id():  # noqa: F811
         return Text
 
+    # =========================================================================
+    # Other methods
+
+    @staticmethod
+    def determine_stix_type(stix_object):
+        if isinstance(stix_object, _DomainObject):
+            return "sdo"
+        elif isinstance(stix_object, _Observable):
+            return "sco"
+        elif isinstance(stix_object, _RelationshipObject):
+            return "sro"
+        elif isinstance(stix_object, _MetaObject):
+            return "common"
+
     @staticmethod
     def array_allowed():
         return False
 
-    def generate_value(self, stix_type, value):
+    def process_value_for_insert(self, stix_type, value):
         sql_type = stix_type.determine_sql_type(self)
         if sql_type == self.determine_sql_type_for_string_property():
             return value

stix2/datastore/relational_db/database_backends/postgres_backend.py

@@ -22,6 +22,9 @@ class PostgresBackend(DatabaseBackend):
     def __init__(self, database_connection_url=default_database_connection_url, force_recreate=False, **kwargs: Any):
         super().__init__(database_connection_url, force_recreate=force_recreate, **kwargs)
 
+    # =========================================================================
+    # schema methods
+
     def _create_schemas(self):
         with self.database_connection.begin() as trans:
             trans.execute(CreateSchema("common", if_not_exists=True))
@@ -48,6 +51,9 @@ def schema_for(stix_class):
     def schema_for_core():
         return "common"
 
+    # =========================================================================
+    # sql type methods (overrides)
+
     @staticmethod
     def determine_sql_type_for_binary_property():  # noqa: F811
         return PostgresBackend.determine_sql_type_for_string_property()
@@ -61,6 +67,9 @@ def determine_sql_type_for_hex_property():  # noqa: F811
     def determine_sql_type_for_timestamp_property():  # noqa: F811
         return TIMESTAMP(timezone=True)
 
+    # =========================================================================
+    # Other methods
+
     @staticmethod
     def array_allowed():
         return True

stix2/datastore/relational_db/input_creation.py

@@ -14,6 +14,22 @@
 from stix2.utils import STIXdatetime
 from stix2.v21.common import KillChainPhase
 
+# =========================================================================
+# generate_insert_information methods
+
+# positional arguments
+#
+#   name:               property name
+#   stix_object:        STIX object data to be inserted in the table
+
+# optional arguments
+#
+#   data_sink:          STIX data sink object
+#   table_name:         name of the related table
+#   schema_name:        name of the schema for the related table, if it exists
+#   parent_table_name:  the name of the parent table, if called for a child table
+#   level:              what "level" of child table is involved
+#   foreign_key_value:
 
 @add_method(Property)
 def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
@@ -41,6 +57,16 @@ def is_valid_type(cls, valid_types):
     return cls in valid_types or instance_in_valid_types(cls, valid_types)
 
 
+def generate_insert_for_dictionary_list(table, next_id, value):
+    insert_stmts = list()
+    for v in value:
+        bindings = dict()
+        bindings["id"] = next_id
+        bindings["value"] = v
+        insert_stmts.append(insert(table).values(bindings))
+    return insert_stmts
+
+
 @add_method(DictionaryProperty)
 def generate_insert_information(self, dictionary_name, stix_object, **kwargs):  # noqa: F811
     bindings = dict()
@@ -60,14 +86,24 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
     # binary, boolean, float, hex,
     # integer, string, timestamp
     valid_types = stix_object._properties[dictionary_name].valid_types
+    child_table_inserts = list()
     for name, value in stix_object[dictionary_name].items():
         bindings = dict()
         if "id" in stix_object:
             bindings["id"] = stix_object["id"]
         elif foreign_key_value:
             bindings["id"] = foreign_key_value
         if not valid_types or len(self.valid_types) == 1:
-            value_binding = "value"
+            if is_valid_type(ListProperty, valid_types):
+                value_binding = "values"
+                if not data_sink.db_backend.array_allowed():
+                    next_id = data_sink.next_id()
+                    table_child = data_sink.tables_dictionary[
+                        canonicalize_table_name(table_name + "_" + dictionary_name + "_" + "values", schema_name)]
+                    child_table_inserts = generate_insert_for_dictionary_list(table_child, next_id, value)
+                    value = next_id
+            else:
+                value_binding = "value"
         elif isinstance(value, int) and is_valid_type(IntegerProperty, valid_types):
             value_binding = "integer_value"
         elif isinstance(value, str) and is_valid_type(StringProperty, valid_types):
@@ -86,6 +122,7 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
 
         insert_statements.append(insert(table).values(bindings))
 
+    insert_statements.extend(child_table_inserts)
     return insert_statements
 
 
@@ -141,7 +178,7 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 @add_method(HexProperty)
 def generate_insert_information(self, name, stix_object, data_sink, **kwargs):  # noqa: F811
-    return {name: data_sink.db_backend.generate_value(self, stix_object[name])}
+    return {name: data_sink.db_backend.process_value_for_insert(self, stix_object[name])}
 
 
 def generate_insert_for_hashes(
@@ -248,7 +285,7 @@ def generate_insert_information(   # noqa: F811
     else:
         if db_backend.array_allowed():
             if isinstance(self.contained, HexProperty):
-                return {name: [data_sink.db_backend.generate_value(self.contained, x) for x in stix_object[name]]}
+                return {name: [data_sink.db_backend.process_value_for_insert(self.contained, x) for x in stix_object[name]]}
             else:
                 return {name: stix_object[name]}
 
@@ -283,6 +320,8 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
     return {name: stix_object[name]}
 
+# =========================================================================
+
 
 def derive_column_name(prop):
     contained_property = prop.contained

stix2/datastore/relational_db/relational_db_testing.py

@@ -7,6 +7,23 @@
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
+email_message = stix2.EmailMessage(
+                    type="email-message",
+                    spec_version="2.1",
+                    id="email-message--0c57a381-2a17-5e61-8754-5ef96efb286c",
+                    from_ref="email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5dda",
+                    to_refs=["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
+                    is_multipart=False,
+                    date="2004-04-19T12:22:23.000Z",
+                    subject="Did you see this?",
+                    additional_header_fields={
+                        "Reply-To": [
+                            "steve@example.com",
+                            "jane@example.com"
+                        ]
+                    }
+)
+
 directory_stix_object = stix2.Directory(
     path="/foo/bar/a",
     path_enc="latin1",
@@ -279,6 +296,10 @@ def main():
 
     if store.sink.db_backend.database_exists:
 
+        x=email_message
+
+        store.add(x)
+
         td = test_dictionary()
 
         store.add(td)