handle timestamp for maria, table creation sorting is correct

stix2/datastore/relational_db/database_backends/database_backend_base.py

@@ -7,6 +7,7 @@
     _DomainObject, _MetaObject, _Observable, _RelationshipObject,
 )
 
+from stix2.properties import HexProperty
 
 class DatabaseBackend:
     def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any):
@@ -128,5 +129,7 @@ def process_value_for_insert(self, stix_type, value):
         sql_type = stix_type.determine_sql_type(self)
         if sql_type == self.determine_sql_type_for_string_property():
             return value
-        elif sql_type == self.determine_sql_type_for_hex_property():
+        elif sql_type == self.determine_sql_type_for_hex_property() and isinstance(stix_type, HexProperty):
             return bytes.fromhex(value)
+        else:
+            return value

stix2/datastore/relational_db/database_backends/mariadb_backend.py

@@ -1,7 +1,7 @@
 import os
 from typing import Any
 
-from sqlalchemy import TIMESTAMP, LargeBinary, Text, VARCHAR
+from sqlalchemy import Text, VARCHAR
 from sqlalchemy.schema import CreateSchema
 
 from stix2.base import (
@@ -69,7 +69,7 @@ def determine_sql_type_for_hex_property():  # noqa: F811
 
     @staticmethod
     def determine_sql_type_for_timestamp_property():  # noqa: F811
-        return TIMESTAMP(timezone=True)
+        return Text
 
     # =========================================================================
     # Other methods
@@ -81,3 +81,5 @@ def array_allowed():
     @staticmethod
     def create_regex_constraint_expression(column_name, pattern):
         return f"{column_name} REGEXP {pattern}"
+
+

stix2/datastore/relational_db/input_creation.py

@@ -1,4 +1,3 @@
-
 from sqlalchemy import insert
 
 from stix2.datastore.relational_db.add_method import add_method
@@ -300,7 +299,7 @@ def generate_insert_information(   # noqa: F811
             for elem in stix_object[name]:
                 bindings = {
                     "id": stix_object["id"],
-                    name: bytes.fromhex(elem) if isinstance(self.contained, HexProperty) else elem,
+                    name: db_backend.process_value_for_insert(self.contained, elem)
                 }
                 insert_statements.append(insert(table).values(bindings))
             return insert_statements
@@ -318,7 +317,9 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 @add_method(TimestampProperty)
 def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
-    return {name: stix_object[name]}
+    db_backend = kwargs["data_sink"].db_backend
+    return {name: db_backend.process_value_for_insert(self, stix_object[name])}
+
 
 # =========================================================================
 
@@ -432,7 +433,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, stix_type_
         if prop_name in core_properties:
             # stored in separate tables below, skip here
             if prop_name not in child_table_properties:
-                core_bindings[prop_name] = value
+                core_bindings[prop_name] = db_backend.process_value_for_insert(stix_object._properties[prop_name], value)
 
     core_insert_statement = insert(core_table).values(core_bindings)
     insert_statements.append(core_insert_statement)

stix2/datastore/relational_db/relational_db_testing.py

@@ -17,7 +17,7 @@
                     from_ref="email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5dda",
                     to_refs=["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
                     is_multipart=False,
-                    date="2004-04-19T12:22:23.000Z",
+                    date="2004-05-19T12:22:23.000Z",
                     subject="Did you see this?",
                     additional_header_fields={
                         "Reply-To": [
@@ -284,13 +284,13 @@ class Test3Class:
 def test_dictionary():
     return Test3Class(
         prop_name={"a": 1, "b": 2.3, "c": "foo"},
-        list_of_timestamps={ "2016-05-12T08:17:27.000Z", "2024-05-12T08:17:27.000Z"}
+        list_of_timestamps=["2016-05-12T08:17:27.000Z", "2024-05-12T08:17:27.000Z"]
     )
 
 
 def main():
     store = RelationalDBStore(
-        MariaDBBackend("mariadb+pymysql://{os.getenv('MARIADB_USER')}:{os.getenv('MARIADB_PASSWORD')}@127.0.0.1:3306/rdb", force_recreate=True),
+        MariaDBBackend(f"mariadb+pymysql://admin:admin@127.0.0.1:3306/rdb", force_recreate=True),
         #PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True),
         #SQLiteBackend("sqlite:///stix-data-sink.db", force_recreate=True),
 
@@ -302,6 +302,9 @@ def main():
 
     if store.sink.db_backend.database_exists:
 
+        ap = kill_chain_test()
+        store.add(ap)
+
         x=email_message
 
         store.add(x)
@@ -325,8 +328,7 @@ def main():
         pdf_file = file_example_with_PDFExt_Object()
         store.add(pdf_file)
 
-        ap = kill_chain_test()
-        store.add(ap)
+
 
         store.add(directory_stix_object)
 

stix2/datastore/relational_db/table_creation.py

@@ -460,7 +460,7 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
                         Column(
                             "values",
                             db_backend.determine_sql_type_for_key_as_int(),
-                            unique = True,
+                            unique=True,
                         ),
                     )
                     child_columns = [
@@ -506,7 +506,8 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
     tables.append(Table(canonicalize_table_name(table_name + "_" + name),
                         metadata,
                         *columns,
-                        UniqueConstraint("id", "name"),
+                        # removed to make sort algorithm work for mariadb
+                        # UniqueConstraint("id", "name"),
                         schema=schema_name))
     return tables
 
@@ -684,7 +685,7 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
     elif ((
         isinstance(
             self.contained,
-            (BinaryProperty, BooleanProperty, StringProperty, IntegerProperty, FloatProperty, HexProperty, TimestampProperty),
+            (BinaryProperty, BooleanProperty, StringProperty, IntegerProperty, FloatProperty, HexProperty, TimestampProperty),  # noqa: E131
         ) and
         not db_backend.array_allowed()
     ) or
@@ -882,12 +883,9 @@ def generate_object_table(
             )
         columns.append(column)
 
-    # all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
-    # all_tables.extend(tables)
-    # return all_tables
-
-    tables.append(Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name))
-    return tables
+    all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
+    all_tables.extend(tables)
+    return all_tables
 
 
 def add_tables(new_tables, tables):